性能检测架构-Doris

准备

  1. macOS

源码编译

  1. 下载仓库
    1
    https://github.com/apache/doris.git
  2. 编译
  • 预安装依赖

    1
    2
    brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \
    python@3 cmake ninja ccache bison byacc gettext wget pcre maven llvm@16 openjdk@17 npm
  • 设置环境变量

    1
    2
    设置JAVA_HOME环境变量,用java17
    设置node环境变量,用node18
  • 修改build.sh

    1
    去掉 -Werror
  • 执行编译

    1
    sh build.sh --broker  --spark-dpp --hive-udf  --be-java-extensions --be --fe

启动

  1. 调大file descriptors
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    # 通过 ulimit 命令调大 file descriptors limit 限制大小
    ulimit -n 65536
    # 查看是否生效
    $ ulimit -n

    # 将该配置写到到启动脚本中,以便下次打开终端会话时不需要再次设置
    # 如果是 bash,执行下面语句
    echo 'ulimit -n 65536' >>~/.bashrc
    # 如果是 zsh,执行下面语句
    echo 'ulimit -n 65536' >>~/.zshrc
  2. 启动BE
    1
    2
    3
    4
    5
    6
    cd output/be/conf
    修改
    priority_networks = 127.0.0.1/32

    cd output/be/bin
    sh ./start_be.sh --daemon
  3. 启动FE
    1
    2
    3
    4
    5
    6
    cd output/fe/conf
    修改
    priority_networks = 127.0.0.1/32

    cd output/fe/bin
    sh ./start_fe.sh --daemon

打开编辑后台

  1. http://127.0.0.1:8030

账号:root
密码为空

  1. 点击playground,注册backend
    1
    2
    3
    4
    ALTER SYSTEM ADD BACKEND "127.0.0.1:9050";
    `

    ## 插入数据
    curl –location-trusted -u root: -H “format:json” -H “read_json_by_line:true” -H “load_to_single_tablet:true” -H “strict_mode:false” -H “timeout:600” -H “expect:100-continue” -T /Users/mac/Documents/Web/doris-master/output/test/log.json http://127.0.0.1:8030/api/log_db/log_table/_stream_load
    1
    2
    3
    4
    5
    6
    7
    8
    9

    ##
    1. 采集 app
    2. 传输 kaffa
    3. 解析 flink
    4. 存储 doris
    5. 展示 grafana

    ## 常见命令
    mysql -uroot -P9030 -h127.0.0.1 -e “show backends;”
    show VARIABLES like ‘%insert%’;
    show partitions from log_table;
    drop table log_table;
    1
    2
    3
    4
    5
    6
    7

    ## 参考
    1. 导入配置参数 [https://doris.apache.org/zh-CN/docs/data-operate/import/import-way/stream-load-manual#%E5%AF%BC%E5%85%A5%E9%85%8D%E7%BD%AE%E5%8F%82%E6%95%B0]

    2. 网易云改造 [https://www.selectdb.com/blog/1369]


    CREATE TABLE jfzfund_998866 (
    application_id VARCHAR(32) NULL COMMENT ‘应用id’,
    log_type VARCHAR(12) NULL COMMENT ‘日志类型/jm/tm’,
    container_id VARCHAR(32) NULL COMMENT ‘container_id’,
    logs_timestamp BIGINT NULL COMMENT ‘日志产生时间’,
    log_level VARCHAR(10) NULL COMMENT ‘日志级别’,
    host_name VARCHAR(32) NULL COMMENT ‘主机名’,
    exception_log TEXT NULL COMMENT ‘异常日志’,
    job_id INT NULL COMMENT ‘任务id’,
    message TEXT NULL COMMENT ‘日志内容’,
    tag TEXT NULL COMMENT ‘日志关键指标’,
    log_file_path TEXT NULL COMMENT ‘日志存储路径’,
    exception_class_name TEXT NULL COMMENT ‘异常类名’,
    exception_type TEXT NULL COMMENT ‘异常类型’,
    exception_message TEXT NULL COMMENT ‘异常msg’,
    exception_caused_by TEXT NULL COMMENT ‘异常caused_by’,
    dt date NULL COMMENT ‘天’,
    hh TEXT NULL COMMENT ‘小时’,
    mm TEXT NULL COMMENT ‘分钟’,
    INDEX idx_exception_message (exception_message) USING INVERTED PROPERTIES(“parser” = “english”),
    INDEX idx_message (message) USING INVERTED PROPERTIES(“parser” = “english”)
    ) ENGINE=OLAP
    DUPLICATE KEY(application_id, log_type, container_id, logs_timestamp, log_level, host_name)
    COMMENT ‘OLAP’
    PARTITION BY RANGE(dt)()
    DISTRIBUTED BY RANDOM BUCKETS 15
    PROPERTIES (
    “replication_num” = “1”,
    “dynamic_partition.enable” = “true”,
    “dynamic_partition.time_unit” = “DAY”,
    “dynamic_partition.start” = “-15”,
    “dynamic_partition.end” = “2”,
    “dynamic_partition.prefix” = “p”,
    “dynamic_partition.buckets” = “15”,
    “dynamic_partition.create_history_partition” = “true”,
    “compression” = “ZSTD”,
    “compaction_policy” = “time_series”
    );