crontab不同机器上的备份
10.8.6.21
#工商主体司法标签
5 * * * * bash /home/collie/product/app_company_tag/deploy/company_risk_tag_daily.sh
# baidu_pc
#*/2 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_baidu_pc_task.yml > /dev/null 2>&1
# baiduv
*/1 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_baiduv_person_task.yml > /dev/null 2>&1
*/1 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_baiduv_company_task.yml > /dev/null 2>&1
# 5118
*/1 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_5118_task.yml > /dev/null 2>&1
# adbug提交任务
# */1 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_adbug_task.yml > /dev/null 2>&1
# adbug从es获取任务提交
30 0 * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_adbug_task_from_es.yml > /dev/null 2>&1
#7icp 爬虫
*/5 7-19 * * * flock -n /home/collie/product/offset_store/7icp-1.lock -c "/home/collie/.conda/envs/collie/bin/python product/app_icp/jobs/data_pump/data_pump.py pump-data --config product/app_icp/data_pump/7icp.yml --default_config product --assign_attrs province province_part1"
*/5 7-19 * * * flock -n /home/collie/product/offset_store/7icp-2.lock -c "/home/collie/.conda/envs/collie/bin/python product/app_icp/jobs/data_pump/data_pump.py pump-data --config product/app_icp/data_pump/7icp.yml --default_config product --assign_attrs province province_part2"
*/5 7-19 * * * flock -n /home/collie/product/offset_store/7icp-3.lock -c "/home/collie/.conda/envs/collie/bin/python product/app_icp/jobs/data_pump/data_pump.py pump-data --config product/app_icp/data_pump/7icp.yml --default_config product --assign_attrs province province_part3"
#zimi 爬虫
#*/5 7-19 * * * flock -n /home/collie/product/offset_store/zimi-1.lock -c "/home/collie/.conda/envs/collie/bin/python product/app_icp/jobs/data_pump/data_pump.py pump-data --config product/app_icp/data_pump/zimi.yml --default_config product --assign_attrs province province_part1"
#*/5 7-19 * * * flock -n /home/collie/product/offset_store/zimi-2.lock -c "/home/collie/.conda/envs/collie/bin/python product/app_icp/jobs/data_pump/data_pump.py pump-data --config product/app_icp/data_pump/zimi.yml --default_config product --assign_attrs province province_part2"
#*/5 7-19 * * * flock -n /home/collie/product/offset_store/zimi-3.lock -c "/home/collie/.conda/envs/collie/bin/python product/app_icp/jobs/data_pump/data_pump.py pump-data --config product/app_icp/data_pump/zimi.yml --default_config product --assign_attrs province province_part3"
# baidu_index
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_index.yml --default_config commit_baidu_index_task
(base) [collie@bdp-server-001 yuanb]$ crontab -l | grep baiduv
# baiduv
*/1 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_baiduv_person_task.yml > /dev/null 2>&1
*/1 * * * * cd /home/collie/product/app_ad/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data -c ../../data_pump/commit_baiduv_company_task.yml > /dev/null 2>&1
10.8.6.23
# 每天5点30
30 5 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_public_company/data_pump/public_company.yml --default_config product
# 税收违法,每周一上午9点
00 09 * * 1 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_tax_punish/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_tax_punish/data_pump/submit_task.yaml --default_config product
# 司法拍卖的爬虫,每隔一天的早晨8:00执行任务
# 0 8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_public_company/data_pump/paimai_rmfysszc.yml --default_config product
# 0 16 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_public_company/data_pump/paimai_rmfysszc.yml --default_config product
# 提交AI竞投用户监测的关键词,每天早晨8点
0 8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/commit_ad_all_task.yml --default_config ad_user_monitor
# 每小时提交一次小微企业
#50 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_xiaowei_spider/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_xiaowei_spider/data_pump/xiaowei.yml
# 全量搜索广告 5000w+
*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/commit_ad_all_task.yml
# 搜索广告筛选关键词1500w+
#*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/commit_ad_all_task.yml --default_config ad_word_performance_2_redis
# 百度规划师
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_planner.yml --default_config commit_baidu_planner_task
# 点睛360
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/dianjing_360.yml --default_config commit_dianjing_360_task
# 点睛360 first
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/dianjing_360_first.yml --default_config commit_dianjing_360_first_task
# 点睛360 third
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/dianjing_360_third.yml --default_config commit_dianjing_360_third_task
# 百度推荐
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_recommend.yml --default_config commit_baidu_recommend_task
# 360趋势-关注度
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/trends_attention.yml --default_config commit_trends_attention_task
# 360趋势-曝光度
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/trends_exposure.yml --default_config commit_trends_exposure_task
# baidu_360合并接口
*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_360.yml --default_config commit_baidu_360_task
# baidu_360 missing_word
# */1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_360.yml --default_config commit_missing_word
# IT桔子每日找新
0 */8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_itjuzi/data_pump/itjuzi_task.yml --default_config itjuzi_init_new_task
# IT桔子持续循环id,比如1~1000w范围内的数据
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_itjuzi/data_pump/itjuzi_old_task.yml --default_config itjuzi_init_old_task
# 每分钟插入1个任务，刷新最新列表newest_event_list
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_itjuzi/data_pump/newest_event.yml --default_config product_init_newest_task
# event_id提交到redis,爬虫会去爬取结果的
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_itjuzi/data_pump/newest_event.yml --default_config event_task_mysql_2_redis
# it桔子 oldest
00 05 * * 1 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_itjuzi/data_pump/oldest_event.yml --default_config product_init_oldest_task
05 05 * * 1 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_itjuzi/data_pump/oldest_event.yml --default_config product_oldest_event_list_2_mysql
05 14 * * 1 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_itjuzi/data_pump/oldest_event.yml --default_config product_oldest_mysql_2_redis
# 小微企业，3分钟导入1次
#*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_xiaowei/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_xiaowei/data_pump/xiaowei.yml
#*/2 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_xiaowei/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_xiaowei/data_pump/xiaowei.yml
#*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_xiaowei/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_xiaowei/data_pump/xiaowei.yml
10 9 * * 1,4 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config AH
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config BJ
10 9 * * 1,4 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config TJ
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HB
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config SX
#10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config NMG
10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config JL
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HLJ
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config SH
#10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config JS
10 10 * * 1 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config ZJ
#10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config FJ
10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config JX
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HEN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HUB
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HUN
10 10 * * 1 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config GD
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HAIN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config CQ
10 9 * * 1,4 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config SC
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config GZ
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config YN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config XZ
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config QH```
10.8.6.47
####
# {minute} {hour} {day-of-month} {month} {day-of-week} {full-path-to-shell-script}
# minute: 区间为 0 – 59
# hour: 区间为0 – 23
# day-of-month: 区间为0 – 31
# month: 区间为1 – 12. 1 是1月. 12是12月.
# Day-of-week: 区间为0 – 7. 周日可以是0或7.
####
# tyc每日例行--工商变更以及关联主体的电话数据推送部署 20201216停止更新:
# 00 01 * * * bash /home/collie/product/app_ic_change_info/deploy/data_pump_push_contact.sh
# tyc每日例行--t_05数据库--招投标
#*/30 * * * * bash /home/collie/product/app_lake/cron/data_pump_tyc_t_05_bid.sh company_bid
# 推送当日新成立企业数据,工作日13点和18点
# 05 13 * * 1,2,3,4,5 bash /home/collie/product/app_ic_change_info/cron/job_one_ftp_hour_13.sh
# 05 18 * * 1,2,3,4,5 bash /home/collie/product/app_ic_change_info/cron/job_one_ftp_hour_18.sh
# 西藏商品爬虫入库(部署为superviror,不用cron)
# 00 06 2,16 * * bash /home/collie/product/app_spider_result/deploy/data_pump_spider_goods.sh
# 对外输出的新成立企业
#*/5 * * * * cd /home/collie/product/app_ic_new/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/all_sync_output.yml
# 新成立企业上线监控
10 9-20 * * * cd /home/collie/product/app_ic_new/monitor_new && /home/collie/.conda/envs/collie/bin/python monitor_ic.py
# 未上线新成立企业提交
10 8,10,15 * * * cd /home/collie/product/app_ic_new/jobs/data_pump/ && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/all_table_crawler_online.yml --default_config old_fail &
# spider--1688_shops(每周二爬一遍所有的店铺)
# 00 09 * * 2 bash /home/collie/product/app_1688/data_pump/spider_job.sh 1688_shops 1>/dev/null
# icp每日例行导出--奇虎
00 01 * * * bash /home/collie/product/app_icp/deploy/icp_data_export_day.sh
# icp每日例行导出--搜狗
05 * * * * bash /home/collie/product/app_icp/deploy/icp_data_export_hour.sh
# 工商api查不到的搜索词提交爬虫
10 * * * * cd /home/collie/product/app_ic_complement/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/api_not_result_keyword.yml
40 * * * * cd /home/collie/product/app_ic_complement/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/api_not_result_keyword.yml --default_config zongju_pc
#国网任务提交
#00 10 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i product/app_sgcc/ansible/hosts.ini product/app_sgcc/ansible/sgcc_kill.yaml product/app_sgcc/ansible/sgcc_run.yaml
#30 10 * * * /home/collie/.conda/envs/collie/bin/python product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task
#00 16 * * * /home/collie/.conda/envs/collie/bin/python product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task_fail
#搜索广告相关
#获取1小时内变化的企业，提交更新统计信息
0 * * * * /home/collie/.conda/envs/collie/bin/python product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config product/app_search_ads/data_pump/adsearch_v2.yml --default_config get_changed_companies
#45 * * * * /home/collie/.conda/envs/collie/bin/python product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config product/app_search_ads/data_pump/adsearch_v2.yml --default_config get_changed_words
#电商数据准备，每隔2小时执行一次
0 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_eb_app/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_eb_app/data_pump/conv_ebusiness_data.yml --default_config 'conv'
#APP数据准备
#小米APP数据，每月15号2点执行一次
0 2 15 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_eb_app/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_eb_app/data_pump/conv_app_data.yml --default_config 'xiaomi'
#华为APP数据,每月15号2点执行一次
0 2 15 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_eb_app/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_eb_app/data_pump/conv_app_data.yml --default_config 'huawei'
# 法院公告数据写文件
05 01 * * * cd /home/collie/product/app_court_notice/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/app_parse_court_notice.yml > court_notice_parse.log 2>&1 &
# 法院公告数据文件入库
05 03 * * * cd /home/collie/product/app_risk/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/risk_court_spider.yml --default_config spider_court_notices_re_to_mysql > court_notice_db.log 2>&1
# 法院公告数据初始解析预警
05 05 * * * bash /home/collie/product/app_risk/deploy/risk_court_notice_warning.sh
# 法院公告数据初始5个json格式网站入库
05 06 * * * cd /home/collie/product/app_risk/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/risk_court_spider.yml --default_config spider_court_notices_re_to_sd_json_web > court_notice_5_json_web.log 2>&1
# 法院公告非json格式网站入库
05 07 * * * cd /home/collie/product/app_risk/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/risk_court_spider.yml --default_config spider_court_notices_re_to_sd_not_json_web > court_notice_not_json_web.log 2>&1
# 开庭公告的网站入库
05 07 * * * cd /home/collie/product/app_risk/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config ../../data_pump/risk_court_spider.yml --default_config spider_court_announcement_re_to_sd > court_announcement.log 2>&1
50 15 * * * bash /home/collie/product/app_mongo_renovate/deploy/renovate_error_credit_no.sh
# 裁判文书数据解析
10 0 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_conv_file/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_conv_file/data_pump/wenshu_conv_file.yml >> /home/collie/product/app_conv_file/data_pump/app_conv_file.log 2>&1
# PDD相似商品数据入库
10 2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_conv_ebusiness/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_conv_ebusiness/data_pump/conv_pdd_reco_goods.yml & >> /home/collie/product/app_conv_ebusiness/data_pump/conv_pdd_reco_goods.log 2>&1
# 司法拍卖解析
55 09 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tb_judicial_sale/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_tb_judicial_sale/data_pump/parse_data.yml
55 09 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tb_judicial_sale/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_tb_judicial_sale/data_pump/parse_add_time.yml
#增量数据同步情况监控
0 1 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_maxwell/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_maxwell/data_pump/update_data_monitor/update_data_monitor.yml
10.8.6.63
#国网任务提交
00 01 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task
40 09 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_python.yaml
45 09 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_cmd.yaml
50 09 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_chrome.yaml
00 10 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_run.yaml
20 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task
00 16 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task_fail
#10 17 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_python.yaml
#13 17 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_cmd.yaml
#16 17 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_chrome.yaml
# 爱企查在营企业4000w+
*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_credit_china/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_aiqicha/data_pump/aiqicha.yml
#天下信用任务提交
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_txxy_app
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_ningbo
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_jiangsu
# 执行公开网任务上传
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_risk_zxgk
#执行信息网站任务提交-补漏
# */10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config redis_risk_pname_id -n 20000
#执行信息网站更新存量-50000
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhixing_update_task
20 09 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhixing_update_task_other
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config shixin_update_task
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhongben_update_task
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config xianxiao_update_task
#执行信息网站找新任务-10000
25 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config shixin_search_task
28 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhongben_search_task
15 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhixing_search_task
15 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config xianxiao_search_task
#法院公告任务
#### 30 12 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml
30 */1 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h1
32 */3 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h3
34 */6 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h6
36 */12 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h12
38 */24 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h24
#55 10 27 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tb_judicial_sale/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_tb_judicial_sale/data_pump/parse_data.yml
#33 15 26 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tb_judicial_sale/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_tb_judicial_sale/data_pump/parse_add_time.yml
#裁判文书爬虫周任务,月任务
0 6 * * 3,7 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_wenshu_task/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_wenshu_task/data_pump/app_wenshu_task.yml --default_config week
0 6 28 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_wenshu_task/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_wenshu_task/data_pump/app_wenshu_task.yml --default_config month
# 拼多多找新任务上���
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_pdd/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_pdd/data_pump/add_recommendation_task.yml --default_config add_recommendation_task
# 高新技术企业
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_innocom/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_innocom/data_dump/innocom.yml --default_config read_from_mysql
# 信用中国双公示任务上传
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_credit_china/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_credit_china/data_pump/push_task.yml
# A级纳税人任务
20 10 15 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_A_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_A_taxpayer/data_pump/push_task.yaml --default_config dev --assign_attrs task_year 2020
# 12315
*/7 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_12315/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_12315/data_dump/P12315.yml
# 环保等级任务
30 8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_environmental_protection_grade/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_environmental_protection_grade/data_pump/normal_list_task.yml
0 12 * * 1,3,5 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_environmental_protection_grade/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_environmental_protection_grade/data_pump/normal_list_task.yml --default_config weekly
# 招投标爬虫任务
15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_bid/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_bid/data_pump/push_task.yml --default_config ccgp_search
15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_bid/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_bid/data_pump/push_task.yml --default_config ggzy_search
# 欠税公告周任务,每周三提交任务
0 0 * * 3 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tax_arrears_announcement/jobs/data_pump/data_pump.py pump-data -c tax_arrears_announcement.yml
10.8.6.27
# >>> qcc >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
## IC失败非1000码处理
### es到sql
20 23 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml
### sql至redis任务
#0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml --default_config tasks
### 临时sql补数据任务
#*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml --default_config temp_tasks
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_retry_task.yml --default_config temp_tasks_err
### 补err_pl产生的fid数据
#0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config err_fid
### 个人详情任务（不工作时保持注释状态）
### tb_err_pl_all表
0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config errpl_all_less_10
10 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config errpl_all_login
10 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config errpl_all_login2
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_err.yml --default_config fid_v3_errpl_all
## 定时vps重新拨号代理至redis(对登录爬虫影响)
50 */3 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/udms/spider_test_consumer/init_tool.py >>/home/collie/tmp/init_vps.log 2>&1
#48 */2 * * * sh /home/collie/sh/reload_qcc_login.sh stop
#52 */2 * * * sh /home/collie/sh/reload_qcc_login.sh start
### changed
0 */2 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_task.yml --default_config changed_fid_pc
0 */4 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_equity_penetration/data_pump/qcc_task.yml --default_config changed_fid_search
0 */2 * * * sh /home/collie/sh/mgr.sh task qcc id
45 * * * * sh /home/collie/sh/mgr.sh task qcc detail_person
45 * * * * sh /home/collie/sh/mgr.sh task qcc detail_person2
0 */3 * * * sh /home/collie/sh/mgr.sh task tyc id
45 */2 * * * sh /home/collie/sh/mgr.sh task tyc detail_person
# <<< qcc <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# >>> general_taxpayer >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# <<< general_taxpayer <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# 卫健委爬虫关停
55 0 * * * sh /home/collie/sh/reload_health.sh stop
5 5 * * * sh /home/collie/sh/reload_health.sh start
# 建筑企业资质证书
0 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_certifications/data_pump/cert.yml --default_config file
# <<< 工商和年报，公告相关任务 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# 工商总局公告爬虫任务
00 11 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_zongju_notice/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_zongju_notice/data_pump/add_spider_task.yml
00 11 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_zongju_notice/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_zongju_notice/data_pump/add_spider_task.yml --default_config product
# 年报任务提交从侦测表
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/ar_task_sql_taskhub.yml --default_config mysql_ar_2021_to_rds
# */10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/ar_task_sql_taskhub.yml --default_config mysql_ar_2021_to_rds_tmp
# 个体户年报侦测和变更：22 from es
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_company_to_task.yaml --default_config product > /dev/null
# 工商例行任务
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task1 > /dev/null
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task2 > /dev/null
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task3 > /dev/null
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task4 > /dev/null
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_pc_spider_push_task.yaml --default_config task5 > /dev/null
# 各省工商任务
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_sichuan
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_zhejiang
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_henan
*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/sichuan/push_task.yaml --default_config task_beijing
# 工商水滴年报侦测上线从文件
#*/15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_shuidi_ar
# 工商行政区可维度任务来源qxb
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/allow_task_sql_to_taskhub.yaml --default_config product
# 工商总局重启
#00 13 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 13 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#00 14 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 14 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#00 15 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 15 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#00 16 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
#30 16 * * * cd /home/collie/deploy_spider;/opt/miniconda3/bin/python spider_admin.py -n zongju_pc -s start -m 5
# 经营异常任务提交，一次性 20220301开始
# */5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/task_sql_to_taskhub.yaml --default_config product
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/zongju_company_to_task.yaml --default_config product
# 吊注销企业例行
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_to_task.yml --default_config product > /dev/null
# 个体户例行更新
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_to_task.yml --default_config product_magor1 > /dev/null
# 工商上线一次性任务
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_pc
# 工商年报上线一次性任务
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_ar_task_sql_taskhub.yml --default_config mysql_ar_2021_to_rds_one_time
# 五一循环任务
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_ks
#*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/one_time_online_task.yaml --default_config product_ks_credit_china
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/task_sql_to_taskhub.yaml --default_config product > /dev/null
# 700万2021成立的缺少2021年报和联系方式一次性侦测任务
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_ic_pc/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic_pc/data_pump/es_to_task.yml --default_config product_es_700w_2021_ar_task > /dev/null
# <<< 工商和年报，公告相关任务 -----------------------------------------------------------------------
# <<< 其他项 <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
# 钉钉企典-在营企业
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_zongju_notice/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_dingdingqidian/data_pump/qidian.yml --default_config product_all_company
# 法院公告庭审直播网站补全
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_tingshen
*/5 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_tingshen_text
# 招投标数据日常任务提交
30 8 * * * bash /home/collie/product/app_bid/deploy/data_pump_search_tasks_submit.sh `date -d '1 days ago' +\%Y-\%m-\%d` 0 0
30 17 * * * bash /home/collie/product/app_bid/deploy/data_pump_search_tasks_submit.sh `date +\%Y-\%m-\%d` 0 0
# 信用中国任务从es
*/4 * * * * cd /home/collie/product/app_credit_china/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config /home/collie/product/app_credit_china/data_pump/push_task.yml > /dev/null 2>&1
# 信用中国例行更新任务
5 * * * * cd /home/collie/product/app_credit_china/jobs/data_pump && /home/collie/.conda/envs/collie/bin/python data_pump.py pump-data --config /home/collie/product/app_credit_china/data_pump/push_task_from_mysql.yml --default_config product
# 香港企业
## 没前缀
0 20 * * 1-6 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_enterprise/data_pump/enterprise_hk.yml
## 带符号前缀
0 20 * * 1-6 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_equity_penetration/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_enterprise/data_pump/enterprise_hk.yml --default_config prefix
yuanbo