crontab不同机器上的备份
10.8.6.23
# 每天5点30
30 5 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_public_company/data_pump/public_company.yml --default_config product
# 税收违法,每周一上午9点
00 09 * * 1 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_tax_punish/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_tax_punish/data_pump/submit_task.yaml --default_config product
# 司法拍卖的爬虫,每隔一天的早晨8:00执行任务
0 8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_public_company/data_pump/paimai_rmfysszc.yml --default_config product
# 提交AI竞投用户监测的关键词,每天早晨8点
0 8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/commit_ad_all_task.yml --default_config ad_user_monitor
# 每小时提交一次小微企业
#50 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_xiaowei_spider/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_xiaowei_spider/data_pump/xiaowei.yml
# 全量搜索广告 5000w+
*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/commit_ad_all_task.yml
# 搜索广告筛选关键词1500w+
#*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/commit_ad_all_task.yml --default_config ad_word_performance_2_redis
# 百度规划师
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_planner.yml --default_config commit_baidu_planner_task
# 点睛360
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/dianjing_360.yml --default_config commit_dianjing_360_task
# 点睛360 first
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/dianjing_360_first.yml --default_config commit_dianjing_360_first_task
# 点睛360 third
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/dianjing_360_third.yml --default_config commit_dianjing_360_third_task
# 百度推荐
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_recommend.yml --default_config commit_baidu_recommend_task
# 360趋势-关注度
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/trends_attention.yml --default_config commit_trends_attention_task
# 360趋势-曝光度
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/trends_exposure.yml --default_config commit_trends_exposure_task
# baidu_360合并接口
*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_360.yml --default_config commit_baidu_360_task
# baidu_360 missing_word
# */1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_search_ads/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_search_ads/data_pump/baidu_360.yml --default_config commit_missing_word
# IT桔子每日找新
0 */8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_itjuzi/data_pump/itjuzi_task.yml --default_config itjuzi_init_new_task
# IT桔子持续循环id,比如1~1000w范围内的数据
*/1 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_itjuzi/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_itjuzi/data_pump/itjuzi_old_task.yml --default_config itjuzi_init_old_task
10 9 * * 1,4 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config AH
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config BJ
10 9 * * 1,4 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config TJ
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HB
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config SX
#10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config NMG
10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config JL
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HLJ
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config SH
#10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config JS
10 10 * * 1 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config ZJ
#10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config FJ
10 10 * * 1,3,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config JX
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HEN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HUB
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HUN
10 10 * * 1 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config GD
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config HAIN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config CQ
10 9 * * 1,4 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config SC
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config GZ
10 10 * * 1,5 cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config YN
10 9 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config XZ
10 10 * * * cd /home/collie && /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_public_company/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_ic/data_pump/general_ic_old.yml --default_config QH
10.8.6.63
#国网任务提交
00 01 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task
40 09 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_python.yaml
45 09 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_cmd.yaml
50 09 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_chrome.yaml
00 10 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_run.yaml
20 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task
00 16 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_sgcc/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_sgcc/data_pump/submit_task.yml --default_config product_submit_task_fail
#10 17 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_python.yaml
#13 17 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_cmd.yaml
#16 17 * * * /home/collie/.conda/envs/collie/bin/ansible-playbook -i /home/collie/product/app_sgcc/ansible/hosts.ini /home/collie/product/app_sgcc/ansible/sgcc_kill_chrome.yaml
# 爱企查在营企业4000w+
*/3 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_credit_china/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_aiqicha/data_pump/aiqicha.yml
#天下信用任务提交
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_txxy_app
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_ningbo
#*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_jiangsu
# 执行公开网任务上传
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_general_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_general_taxpayer/data_pump/general_taxpayer_push_task.yml -n 20000 --default_config pump_risk_zxgk
#执行信息网站任务提交-补漏
# */10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config redis_risk_pname_id -n 20000
#执行信息网站更新存量-50000
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhixing_update_task
20 09 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhixing_update_task_other
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config shixin_update_task
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhongben_update_task
*/20 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config xianxiao_update_task
#执行信息网站找新任务-10000
25 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config shixin_search_task
28 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhongben_search_task
15 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config zhixing_search_task
15 10 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_risk_zxgk/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_risk_zxgk/data_pump/app_risk_zxgk.yaml --default_config xianxiao_search_task
#法院公告任务
#### 30 12 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml
30 */1 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h1
32 */3 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h3
34 */6 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h6
36 */12 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h12
38 */24 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_court_notice/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_court_notice/data_pump/app_court_notice_task.yml --default_config task_h24
#55 10 27 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tb_judicial_sale/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_tb_judicial_sale/data_pump/parse_data.yml
#33 15 26 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tb_judicial_sale/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_tb_judicial_sale/data_pump/parse_add_time.yml
#裁判文书爬虫周任务,月任务
0 6 * * 3,7 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_wenshu_task/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_wenshu_task/data_pump/app_wenshu_task.yml --default_config week
0 6 28 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_wenshu_task/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_wenshu_task/data_pump/app_wenshu_task.yml --default_config month
# 拼多多找新任务上���
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_pdd/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_pdd/data_pump/add_recommendation_task.yml --default_config add_recommendation_task
# 高新技术企业
*/30 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_innocom/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_innocom/data_dump/innocom.yml --default_config read_from_mysql
# 信用中国双公示任务上传
*/10 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_credit_china/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_credit_china/data_pump/push_task.yml
# A级纳税人任务
20 10 15 * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_A_taxpayer/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_A_taxpayer/data_pump/push_task.yaml --default_config dev --assign_attrs task_year 2020
# 12315
*/7 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_12315/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_12315/data_dump/P12315.yml
# 环保等级任务
30 8 * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_environmental_protection_grade/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_environmental_protection_grade/data_pump/normal_list_task.yml
0 12 * * 1,3,5 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_environmental_protection_grade/jobs/data_pump/data_pump.py pump-data --config /home/collie/product/app_environmental_protection_grade/data_pump/normal_list_task.yml --default_config weekly
# 招投标爬虫任务
15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_bid/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_bid/data_pump/push_task.yml --default_config ccgp_search
15 * * * * /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_bid/jobs/data_pump/data_pump.py pump-data -c /home/collie/product/app_bid/data_pump/push_task.yml --default_config ggzy_search
# 欠税公告周任务,每周三提交任务
0 0 * * 3 /home/collie/.conda/envs/collie/bin/python /home/collie/product/app_tax_arrears_announcement/jobs/data_pump/data_pump.py pump-data -c tax_arrears_announcement.yml
yuanbo