allucanget/jobs
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

do not remove stale urls

Browse Source
This commit is contained in:
zwitschi
2025-09-17 15:30:29 +02:00
parent e26dc9c164
commit 39900ea564
1 changed files with 2 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
web/craigslist.py
View File

@@ -94,19 +94,8 @@ def fetch_listings():
fetched_at=datetime.now(timezone.utc), fetched_at=datetime.now(timezone.utc),
) )
# Remove stale listings: those present in DB but not discovered now. yield f"Listing fetch complete: {len(discovered_urls)} discovered, {len(new_rows)} new,\n"
stale_urls = existing_db_urls - discovered_urls return {"discovered": len(discovered_urls), "new": len(new_rows)}
if stale_urls:
yield f"Removing {len(stale_urls)} stale listings...\n"
for url in stale_urls:
try:
jid = url_to_job_id(url)
db_delete_job(jid)
except Exception:
pass
yield f"Listing fetch complete: {len(discovered_urls)} discovered, {len(new_rows)} new, {len(stale_urls)} stale\n"
return {"discovered": len(discovered_urls), "new": len(new_rows), "stale": len(stale_urls)}
def process_job_url(job_url: str): def process_job_url(job_url: str):