科创吧

准备

1. macOS

源码编译

1. 下载仓库

   https://github.com/apache/doris.git

2. 编译

• 预安装依赖

  brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \
  python@3 cmake ninja ccache bison byacc gettext wget pcre maven llvm@16 openjdk@17 npm

• 设置环境变量

  设置JAVA_HOME环境变量，用java17
  设置node环境变量，用node18

• 修改build.sh

  去掉 -Werror

• 执行编译

  sh build.sh --broker  --spark-dpp --hive-udf  --be-java-extensions --be --fe

启动

1. 调大file descriptors

   # 通过 ulimit 命令调大 file descriptors limit 限制大小
   ulimit -n 65536
   # 查看是否生效
   $ ulimit -n
   
   # 将该配置写到到启动脚本中，以便下次打开终端会话时不需要再次设置
   # 如果是 bash，执行下面语句
   echo 'ulimit -n 65536' >>~/.bashrc
   # 如果是 zsh，执行下面语句
   echo 'ulimit -n 65536' >>~/.zshrc

2. 启动BE

   cd output/be/conf
   修改
   priority_networks = 127.0.0.1/32
   
   cd output/be/bin
   sh ./start_be.sh --daemon

3. 启动FE

   cd output/fe/conf
   修改
   priority_networks = 127.0.0.1/32
   
   cd output/fe/bin
   sh ./start_fe.sh --daemon

打开编辑后台

1. http://127.0.0.1:8030

账号:root
密码为空

2. 点击playground，注册backend

   ALTER SYSTEM ADD BACKEND "127.0.0.1:9050";
   `
   
   ## 插入数据

   curl –location-trusted -u root: -H “format:json” -H “read_json_by_line:true” -H “load_to_single_tablet:true” -H “strict_mode:false” -H “timeout:600” -H “expect:100-continue” -T /Users/mac/Documents/Web/doris-master/output/test/log.json http://127.0.0.1:8030/api/log_db/log_table/_stream_load

   ## 
   1. 采集 app
   2. 传输 kaffa
   3. 解析 flink
   4. 存储 doris
   5. 展示 grafana
   
   ## 常见命令

   mysql -uroot -P9030 -h127.0.0.1 -e “show backends;”
   show VARIABLES like ‘%insert%’;
   show partitions from log_table;
   drop table log_table;

   ## 参考
   1. 导入配置参数 [https://doris.apache.org/zh-CN/docs/data-operate/import/import-way/stream-load-manual#%E5%AF%BC%E5%85%A5%E9%85%8D%E7%BD%AE%E5%8F%82%E6%95%B0]
   
   2. 网易云改造 [https://www.selectdb.com/blog/1369]

   CREATE TABLE jfzfund_998866 (
   application_id VARCHAR(32) NULL COMMENT ‘应用id’,
   log_type VARCHAR(12) NULL COMMENT ‘日志类型/jm/tm’,
   container_id VARCHAR(32) NULL COMMENT ‘container_id’,
   logs_timestamp BIGINT NULL COMMENT ‘日志产生时间’,
   log_level VARCHAR(10) NULL COMMENT ‘日志级别’,
   host_name VARCHAR(32) NULL COMMENT ‘主机名’,
   exception_log TEXT NULL COMMENT ‘异常日志’,
   job_id INT NULL COMMENT ‘任务id’,
   message TEXT NULL COMMENT ‘日志内容’,
   tag TEXT NULL COMMENT ‘日志关键指标’,
   log_file_path TEXT NULL COMMENT ‘日志存储路径’,
   exception_class_name TEXT NULL COMMENT ‘异常类名’,
   exception_type TEXT NULL COMMENT ‘异常类型’,
   exception_message TEXT NULL COMMENT ‘异常msg’,
   exception_caused_by TEXT NULL COMMENT ‘异常caused_by’,
   dt date NULL COMMENT ‘天’,
   hh TEXT NULL COMMENT ‘小时’,
   mm TEXT NULL COMMENT ‘分钟’,
   INDEX idx_exception_message (exception_message) USING INVERTED PROPERTIES(“parser” = “english”),
   INDEX idx_message (message) USING INVERTED PROPERTIES(“parser” = “english”)
   ) ENGINE=OLAP
   DUPLICATE KEY(application_id, log_type, container_id, logs_timestamp, log_level, host_name)
   COMMENT ‘OLAP’
   PARTITION BY RANGE(dt)()
   DISTRIBUTED BY RANDOM BUCKETS 15
   PROPERTIES (
   “replication_num” = “1”,
   “dynamic_partition.enable” = “true”,
   “dynamic_partition.time_unit” = “DAY”,
   “dynamic_partition.start” = “-15”,
   “dynamic_partition.end” = “2”,
   “dynamic_partition.prefix” = “p”,
   “dynamic_partition.buckets” = “15”,
   “dynamic_partition.create_history_partition” = “true”,
   “compression” = “ZSTD”,
   “compaction_policy” = “time_series”
   );