第一个参数:initScans(job)
第一个参数:initScans(job)
private List<Scan> initScans(Job job) {
// 时间戳+....
Configuration conf = job.getConfiguration();
// 获取运行时间: yyyy-MM-dd
String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES);
long startDate = TimeUtil.parseString2Long(date);
long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS;
Scan scan = new Scan();
// 定义hbase扫描的开始rowkey和结束rowkey
scan.setStartRow(Bytes.toBytes("" + startDate));
scan.setStopRow(Bytes.toBytes("" + endDate));
FilterList filterList = new FilterList();
// 过滤数据,只分析launch事件
filterList.addFilter(new SingleColumnValueFilter(Bytes
.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME), Bytes
.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME),
CompareOp.EQUAL, Bytes.toBytes(EventEnum.LAUNCH.alias)));
// 定义mapper中需要获取的列名
String[] columns = new String[] {
EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME,
EventLogConstants.LOG_COLUMN_NAME_UUID,
EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME,
EventLogConstants.LOG_COLUMN_NAME_PLATFORM,
EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME,
EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION };
// scan.addColumn(family, qualifier)
filterList.addFilter(this.getColumnFilter(columns));
scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME,
Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS));
scan.setFilter(filterList);
return Lists.newArrayList(scan);
}
获取配置
Configuration conf = job.getConfiguration();
// 获取运行时间: yyyy-MM-dd processArgs方法里面的conf.set
String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES);
将字符转换成long类型的时间 TimeUtil类完整方法
long startDate = TimeUtil.parseString2Long(date);
long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS;
DAY_OF_MILLISECONDS 一天有多少个毫秒
Scan scan = new Scan(); // 定义hbase扫描的开始rowkey和结束rowkey scan.setStartRow(Bytes.toBytes("" + startDate)); scan.setStopRow(Bytes.toBytes("" + endDate));
FilterList filterList = new FilterList();
// 过滤数据,只分析launch事件
FilterList 类的实现
final public class FilterList extends Filter public abstract class Filter
public void addFilter(Filter filter)
// 过滤数据,只分析launch事件
filterList.addFilter(new SingleColumnValueFilter(Bytes
.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME), Bytes
.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME),
CompareOp.EQUAL, Bytes.toBytes(EventEnum.LAUNCH.alias)));
filterList.addFilter(new SingleColumnValueFilter(
/**
- Constructor for binary compare of the value of a single column. If the
- column is found and the condition passes, all columns of the row will be
- emitted. If the condition fails, the row will not be emitted.
- <p>
- Use the filterIfColumnMissing flag to set whether the rest of the columns
- in a row will be emitted if the specified column to check is not found in
- the row.
- @param family name of column family
- @param qualifier name of column qualifier
- @param compareOp operator
- @param value value to compare column values against */ public SingleColumnValueFilter(final byte [] family, final byte [] qualifier, final CompareOp compareOp, final byte[] value) { this(family, qualifier, compareOp, new BinaryComparator(value)); } }
Bytes.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME),
EventLogConstants
/**
* event_logs表的列簇名称
*/
public static final String EVENT_LOGS_FAMILY_NAME = "log";
Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME),
/** * 事件名称 */ public static final String LOG_COLUMN_NAME_EVENT_NAME = "en";
CompareOp.EQUAL,
是在CompareFilter 中定义的
public abstract class Filter { public abstract class FilterBase extends Filter { public abstract class CompareFilter extends FilterBase {
public abstract class CompareFilter extends FilterBase {
/** Comparison operators. / @InterfaceAudience.Public @InterfaceStability.Stable public enum CompareOp { /* less than / LESS, /* less than or equal to / LESS_OR_EQUAL, /* equals / EQUAL, /* not equal / NOT_EQUAL, /* greater than or equal to / GREATER_OR_EQUAL, /* greater than / GREATER, /* no operation */ NO_OP, }
Bytes.toBytes(EventEnum.LAUNCH
/** * 事件枚举类。指定事件的名称 * * @author root * */ public static enum EventEnum { LAUNCH(1, "launch event", "e_l"), // launch事件,表示第一次访问 PAGEVIEW(2, "page view event", "e_pv"), // 页面浏览事件 CHARGEREQUEST(3, "charge request event", "e_crt"), // 订单生产事件 CHARGESUCCESS(4, "charge success event", "e_cs"), // 订单成功支付事件 CHARGEREFUND(5, "charge refund event", "e_cr"), // 订单退款事件 EVENT(6, "event duration event", "e_e") // 事件 ;
public final int id; // id 唯一标识
public final String name; // 名称
public final String alias; // 别名,用于数据收集的简写
// 定义mapper中需要获取的列名 String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME, //事件名称en EventLogConstants.LOG_COLUMN_NAME_UUID, //用户UUID u_ud EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, //服务器时间s_time EventLogConstants.LOG_COLUMN_NAME_PLATFORM, //用户平台pl EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, //browser EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION //browser_v };
filterList.addFilter(this.getColumnFilter(columns));
/** * 获取这个列名过滤的column * * @param columns * @return */ private Filter getColumnFilter(String[] columns) { int length = columns.length; byte[][] filter = new byte[length][]; for (int i = 0; i < length; i++) { filter[i] = Bytes.toBytes(columns[i]); } return new MultipleColumnPrefixFilter(filter); }
>scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME,
// If an application wants to use multiple scans over different tables each scan must // define this attribute with the appropriate table name by calling // scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName)) static public final String SCAN_ATTRIBUTES_TABLE_NAME = "scan.attributes.table.name";
>Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS));
/** * 表名称 */ public static final String HBASE_NAME_EVENT_LOGS = "eventlog";
scan.setFilter(filterList);
return Lists.newArrayList(scan);
}
项目入口
http://www.jianshu.com/p/aa3b12b0d426
- Linux在批量服务器管理中实用的PS1命令提示符格式
- Golang语言社区-并发模型和应用场景
- 翻过那座山,就能看见海|kubernetes让DBA更优雅地管理数据库
- 基于TextCNN的谩骂评论识别模型
- Golang语言社区-文件操作
- MySQL service启动脚本浅析(r12笔记第59天)
- 分享WordPress显示评论者IP的归属地及运营商信息的2种方案
- 解决dos2unix/unix2dos报错,并在家目录下生成u2dtmp*文件问题
- 【前沿】TensorFlow Pytorch Keras代码实现深度学习大神Hinton NIPS2017 Capsule论文
- Nginx通过二级目录(路径)映射不同的反向代理,规避IP+端口访问
- MySQL中批量初始化数据的对比测试(r12笔记第71天)
- Golang语言--包的概念、导入与可见性
- MySQL中的change,modify和自增列的关系(r12笔记第70天)
- Golang 值得注意的地方
- JavaScript 教程
- JavaScript 编辑工具
- JavaScript 与HTML
- JavaScript 与Java
- JavaScript 数据结构
- JavaScript 基本数据类型
- JavaScript 特殊数据类型
- JavaScript 运算符
- JavaScript typeof 运算符
- JavaScript 表达式
- JavaScript 类型转换
- JavaScript 基本语法
- JavaScript 注释
- Javascript 基本处理流程
- Javascript 选择结构
- Javascript if 语句
- Javascript if 语句的嵌套
- Javascript switch 语句
- Javascript 循环结构
- Javascript 循环结构实例
- Javascript 跳转语句
- Javascript 控制语句总结
- Javascript 函数介绍
- Javascript 函数的定义
- Javascript 函数调用
- Javascript 几种特殊的函数
- JavaScript 内置函数简介
- Javascript eval() 函数
- Javascript isFinite() 函数
- Javascript isNaN() 函数
- parseInt() 与 parseFloat()
- escape() 与 unescape()
- Javascript 字符串介绍
- Javascript length属性
- javascript 字符串函数
- Javascript 日期对象简介
- Javascript 日期对象用途
- Date 对象属性和方法
- Javascript 数组是什么
- Javascript 创建数组
- Javascript 数组赋值与取值
- Javascript 数组属性和方法