yuanbo基本信息
法院公告爬虫
每天轮询所有已知的网站的某些页面，爬到的数据先存到文件，后续进行解析
数据名称（中文）
法院公告信息数据英文名称
risk_court_notice采集网站（采集入口）
网站数据接口:
http://www.guizhoucourt.gov.cn/
有很多采集频率及采集策略
存量更新策略
存量无需更新
增量按照网站的翻页数进行遍历更新
增量采集策略
增量按照网站的翻页取最新的几页爬虫
责任人
杨龙斌爬虫名称
risk_court_notice代码地址
项目地址:http://tech.pingansec.com/granite/project-judicature/-/tree/dev_court_notice队列名称及队列地址
- redis host: redis://:utn@0818@bdp-mq-001.redis.rds.aliyuncs.com:6379/7
- redis port: 6379
- redis db: 7
- redis key:
- risk_court_notice
优先级队列说明
任务来源
待完善任务样例
任务参数说明
data_type说明
爬虫结果的超级数据
实际爬虫结果的数据结构
{
"data": {
"court_items": [
{
"source": "http://www.cqfygzfw.com",
"data": {
"count": 2266,
"prePage": 189,
"nextPage": 191,
"currentPage": 190,
"pageSize": 10,
"pageCount": 227,
"data": [
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M57a8543afa-58b6-424b-b031-b150762a11f7",
"ajbs": "M5727b30594-e934-4439-8c7d-5216cf4f3c92",
"ahqc": "(2021)渝0116民初8949号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日09:30到10:00在第四人民法庭二审判法庭开庭审理(2021)渝0116民初8949号原告罗远林诉被告王开碧,杨军房屋买卖合同纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M5748e4ad11-49ae-4086-98ad-7ee6701e9da6",
"ajbs": "M5725653880-7d72-4636-87b6-06bb9067aaad",
"ahqc": "(2021)渝0116民初11913号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日14:30到15:30在第三人民法庭一审判法庭开庭审理(2021)渝0116民初11913号原告黄来毅诉被告周胜辉,高进民间借贷纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M57dfd4ebbd-88e8-40a6-9c89-fc732aa92eac",
"ajbs": "M57283867b8-a85b-41ba-80f4-14d6b0324173",
"ahqc": "(2021)渝0116民初9532号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日16:00到17:30在第九审判法庭开庭审理(2021)渝0116民初9532号原告蒲德琼诉被告重庆市江津区东风装饰工程有限公司,翁昌纯民间借贷纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M571533c068-5b80-4e98-832d-0dba1e2d9714",
"ajbs": "0c2a2c55-c6a1-4c72-916f-747011d9af25",
"ahqc": "(2021)渝0116民初9784号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日09:30到12:00在第二十二审判法庭开庭审理(2021)渝0116民初9784号原告孙兴福诉被告深圳市宝安区杨静便利店产品责任纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M577b4ea5d4-dc70-4bf5-8d8e-4f5a5e41a16c",
"ajbs": "fba8d204-1b20-4d15-88a0-765c339a4d99",
"ahqc": "(2021)渝0116民初10444号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日14:30到15:30在第十审判法庭开庭审理(2021)渝0116民初10444号原告中国农业银行股份有限公司重庆江津支行诉被告代龑信用卡纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M57bbd9cf46-6f17-43c3-b634-e4605cb8cad0",
"ajbs": "79559639-d046-4f56-91d6-67ec7c9142ed",
"ahqc": "(2021)渝0116民初10431号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日10:00到11:00在第十四审判法庭开庭审理(2021)渝0116民初10431号原告中国农业银行股份有限公司重庆江津支行诉被告樊涛信用卡纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M576399da41-7dc5-483f-b646-bf9d923bece2",
"ajbs": "M57f0d527a0-8bde-4d47-8c68-39abe3c20746",
"ahqc": "(2021)渝0116民初10243号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日09:30到12:00在第二十六审判法庭开庭审理(2021)渝0116民初10243号原告李兴诉被告中建致弘重庆建设开发有限公司,德尔未来科技控股集团股份有限公司,重庆合亨装饰工程有限公司,重庆鼎用装饰工程有限公司劳动争议一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M57f930f5d2-b593-491c-b7dd-e0acf4e5f49e",
"ajbs": "8d4c3560-dc86-4c7b-9293-b92100806fcb",
"ahqc": "(2021)渝0116民初10256号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日14:30到15:00在第九审判法庭开庭审理(2021)渝0116民初10256号原告重庆天旗实业有限公司诉被告重庆正商实业有限公司,李秋忱买卖合同纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M572696f301-3530-47c1-b128-bead5db7fa81",
"ajbs": "e07c3d44-a0b0-4bf7-9c45-5decb09d658d",
"ahqc": "(2021)渝0116民初10962号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日09:30到12:00在第六审判法庭开庭审理(2021)渝0116民初10962号原告重庆市江津区鹏程物业管理有限公司诉被告曾维财物业服务合同纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
},
{
"count": 0,
"pageSize": 10,
"pageCount": 0,
"page": 1,
"activePage": false,
"start": 0,
"end": 10,
"startDate": null,
"endDate": null,
"dateColName": null,
"retCode": 0,
"retMsg": null,
"bsdf": null,
"cbbm": null,
"cbrmc": null,
"pflag": false,
"sqlServec": 0,
"id": "M5799819ada-d1ea-4d80-909f-6fb9d66d4d19",
"ajbs": "f4b3a0e1-8aaf-4ab5-a920-6d6163d2b60d",
"ahqc": "(2021)渝0116民初10578号",
"ndh": null,
"bzzh": null,
"ajbh": null,
"ktxh": null,
"ggdw": "重庆市江津区人民法院",
"gglx": null,
"gglxmc": null,
"ggnr": "本院定于二〇二一年七月二十七日15:00到17:30在第十一审判法庭开庭审理(2021)渝0116民初10578号原告万世友诉被告敬敏抚养费纠纷一案。",
"fbrxm": null,
"ggrq": 1627315200000,
"xsksrq": null,
"xsjsrq": null,
"xsmtkssj": "08:30",
"xsmtjssj": null,
"xsms": null,
"ggdpbz": null,
"zzrbm": null,
"zzrq": null,
"qybz": null,
"xssx": null,
"sm": null,
"sysTime": null,
"fydm": null,
"fymc": "重庆市江津区人民法院",
"fydms": null,
"startdate": null,
"enddate": null
}
]
}
}
]
},
"http_code": 200,
"error_msg": "",
"task_result": 1000,
"data_type": "detail",
"spider_start_time": "2021-07-27 13:43:15.442",
"spider_end_time": "2021-07-27 13:43:16",
"task_params": {},
"metadata": {},
"spider_name": "court_notice",
"spider_ip": "192.168.56.1"
}爬虫运行环境
scrapy爬虫部署信息
机器：node_51
项目名称：judicature_spiderTaskhub地址
Taskhub调度规则说明
task_result=1000 # 正常获取到详情任务
task_result=1101 # 无结果信息,表示官网查不到这个数据,同样需要解析入库
task_result=9101 # 超时错误，需要进行重试，目前重试5次
task_result=8000 # 参数错误爬虫监控指标设计
(先观察，待补充)
索引:
监控频率:
监控起止时间:
报警条件:
报警群:
报警内容:
数据归集
责任人
数据归集方式
-
爬虫直接写kafka
-
爬虫写文件logstash采集
爬虫结果目录
/data/judicature_spiders/court_notice归集后存放目录
logstash配置文件名称
logstash文件采集type
数据归集的topic
topic_id => "general-taxpayer"ES日志索引及筛选条件
监控指标看板
数据保留策略
数据清洗
责任人
代码地址
部署地址
部署方法及说明
- crontab + data_pump
- supervisor + data_pump
- supervisor + consumer
数据接收来源
数据存储表地址
- 数据库地址：
- 表名：