准备
- macOS
源码编译
- 下载仓库
1
https://github.com/apache/doris.git
- 编译
预安装依赖
1
2brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \
python@3 cmake ninja ccache bison byacc gettext wget pcre maven llvm@16 openjdk@17 npm设置环境变量
1
2设置JAVA_HOME环境变量,用java17
设置node环境变量,用node18修改build.sh
1
去掉 -Werror
执行编译
1
sh build.sh --broker --spark-dpp --hive-udf --be-java-extensions --be --fe
启动
- 调大file descriptors
1
2
3
4
5
6
7
8
9
10# 通过 ulimit 命令调大 file descriptors limit 限制大小
ulimit -n 65536
# 查看是否生效
$ ulimit -n
# 将该配置写到到启动脚本中,以便下次打开终端会话时不需要再次设置
# 如果是 bash,执行下面语句
echo 'ulimit -n 65536' >>~/.bashrc
# 如果是 zsh,执行下面语句
echo 'ulimit -n 65536' >>~/.zshrc - 启动BE
1
2
3
4
5
6cd output/be/conf
修改
priority_networks = 127.0.0.1/32
cd output/be/bin
sh ./start_be.sh --daemon - 启动FE
1
2
3
4
5
6cd output/fe/conf
修改
priority_networks = 127.0.0.1/32
cd output/fe/bin
sh ./start_fe.sh --daemon
打开编辑后台
账号:root
密码为空
- 点击playground,注册backendcurl –location-trusted -u root: -H “format:json” -H “read_json_by_line:true” -H “load_to_single_tablet:true” -H “strict_mode:false” -H “timeout:600” -H “expect:100-continue” -T /Users/mac/Documents/Web/doris-master/output/test/log.json http://127.0.0.1:8030/api/log_db/log_table/_stream_load
1
2
3
4ALTER SYSTEM ADD BACKEND "127.0.0.1:9050";
`
## 插入数据mysql -uroot -P9030 -h127.0.0.1 -e “show backends;”1
2
3
4
5
6
7
8
9
##
1. 采集 app
2. 传输 kaffa
3. 解析 flink
4. 存储 doris
5. 展示 grafana
## 常见命令
show VARIABLES like ‘%insert%’;
show partitions from log_table;
drop table log_table;CREATE TABLE jfzfund_998866 (1
2
3
4
5
6
7
## 参考
1. 导入配置参数 [https://doris.apache.org/zh-CN/docs/data-operate/import/import-way/stream-load-manual#%E5%AF%BC%E5%85%A5%E9%85%8D%E7%BD%AE%E5%8F%82%E6%95%B0]
2. 网易云改造 [https://www.selectdb.com/blog/1369]
application_id VARCHAR(32) NULL COMMENT ‘应用id’,
log_type VARCHAR(12) NULL COMMENT ‘日志类型/jm/tm’,
container_id VARCHAR(32) NULL COMMENT ‘container_id’,
logs_timestamp BIGINT NULL COMMENT ‘日志产生时间’,
log_level VARCHAR(10) NULL COMMENT ‘日志级别’,
host_name VARCHAR(32) NULL COMMENT ‘主机名’,
exception_log TEXT NULL COMMENT ‘异常日志’,
job_id INT NULL COMMENT ‘任务id’,
message TEXT NULL COMMENT ‘日志内容’,
tag TEXT NULL COMMENT ‘日志关键指标’,
log_file_path TEXT NULL COMMENT ‘日志存储路径’,
exception_class_name TEXT NULL COMMENT ‘异常类名’,
exception_type TEXT NULL COMMENT ‘异常类型’,
exception_message TEXT NULL COMMENT ‘异常msg’,
exception_caused_by TEXT NULL COMMENT ‘异常caused_by’,
dt date NULL COMMENT ‘天’,
hh TEXT NULL COMMENT ‘小时’,
mm TEXT NULL COMMENT ‘分钟’,
INDEX idx_exception_message (exception_message) USING INVERTED PROPERTIES(“parser” = “english”),
INDEX idx_message (message) USING INVERTED PROPERTIES(“parser” = “english”)
) ENGINE=OLAP
DUPLICATE KEY(application_id, log_type, container_id, logs_timestamp, log_level, host_name)
COMMENT ‘OLAP’
PARTITION BY RANGE(dt)()
DISTRIBUTED BY RANDOM BUCKETS 15
PROPERTIES (
“replication_num” = “1”,
“dynamic_partition.enable” = “true”,
“dynamic_partition.time_unit” = “DAY”,
“dynamic_partition.start” = “-15”,
“dynamic_partition.end” = “2”,
“dynamic_partition.prefix” = “p”,
“dynamic_partition.buckets” = “15”,
“dynamic_partition.create_history_partition” = “true”,
“compression” = “ZSTD”,
“compaction_policy” = “time_series”
);