🆕尝试使用脚本实现定时任务和api

This commit is contained in:
2024-08-03 12:31:41 +08:00
parent f9fddaa63e
commit 8afdc06741
4 changed files with 56 additions and 44 deletions

View File

@ -24,31 +24,9 @@ app.add_middleware(
)
# 配置日志记录
log_file = "grab.log"
log_file = "cron_grab"
logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
data_lock = Lock()
def fetch_articles():
logging.info("开始抓取文章...")
try:
config = load_config("./conf.yaml")
if config["spider_settings"]["enable"]:
json_url = config['spider_settings']['json_url']
article_count = config['spider_settings']['article_count']
logging.info(f"正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章")
result, errors = fetch_and_process_data(json_url=json_url, count=article_count)
sorted_result = sort_articles_by_time(result)
with open("all.json", "w", encoding="utf-8") as f:
json.dump(sorted_result, f, ensure_ascii=False, indent=2)
with open("errors.json", "w", encoding="utf-8") as f:
json.dump(errors, f, ensure_ascii=False, indent=2)
logging.info("文章抓取成功")
else:
logging.warning("抓取设置在配置中被禁用。")
except Exception as e:
logging.error(f"抓取文章时出错: {e}")
@app.get("/", response_class=HTMLResponse)
async def root():
try:
@ -95,24 +73,11 @@ async def get_random_article():
except json.JSONDecodeError:
return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500)
def schedule_tasks():
schedule.every(4).hours.do(fetch_articles)
while True:
schedule.run_pending()
time.sleep(1)
if __name__ == '__main__':
# 清空日志文件
if os.path.exists(log_file):
with open(log_file, 'w'):
pass
fetch_articles() # 启动时立即抓取一次
# 启动调度任务线程
task_thread = Thread(target=schedule_tasks)
task_thread.start()
# 启动FastAPI应用
import uvicorn
uvicorn.run(app, host='0.0.0.0', port=1223)