allucanget/jobs
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reorganize imports

Browse Source 
removing unused imports
This commit is contained in:
zwitschi
2025-09-17 17:12:16 +02:00
parent e549fae3f6
commit 2ae1e2058d
2 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
web/craigslist.py
View File

@@ -15,6 +15,7 @@ from web.db import (
# Import utility functions # Import utility functions
from web.utils import ( from web.utils import (
get_base_url,
make_request_with_retry, make_request_with_retry,
now_iso, now_iso,
) )
@@ -54,9 +55,10 @@ def fetch_listings():
if not keyword_name: if not keyword_name:
continue continue
# Build a canonical search identifier for this region+keyword combination. # Build a canonical search identifier for this region+keyword combination.
url = get_base_url().format(region=region, keyword=keyword_name.replace(" ", "+"))
search_page_id = f"search:{region_name}:{keyword_name}" search_page_id = f"search:{region_name}:{keyword_name}"
try: try:
last = get_last_fetch_time(search_page_id) last = get_last_fetch_time(url)
if last is not None: if last is not None:
# skip if fetched within the last 24 hours # skip if fetched within the last 24 hours
age = datetime.now( age = datetime.now(
@@ -72,7 +74,7 @@ def fetch_listings():
yield f"Processing {region_name} + {keyword_name} ({processed}/{total_combinations})...\n" yield f"Processing {region_name} + {keyword_name} ({processed}/{total_combinations})...\n"
# record that we're fetching this search page now # record that we're fetching this search page now
try: try:
insert_log(search_page_id, region=region_name, insert_log(url, region=region_name,
keyword=keyword_name, fetched_at=datetime.now(timezone.utc)) keyword=keyword_name, fetched_at=datetime.now(timezone.utc))
except Exception: except Exception:
pass pass
@@ -99,6 +101,15 @@ def fetch_listings():
def process_job_url(job_url: str, region: str = "", keyword: str = ""): def process_job_url(job_url: str, region: str = "", keyword: str = ""):
last = get_last_fetch_time(job_url)
if last is not None:
# skip if fetched within the last 24 hours
age = datetime.now(
timezone.utc) - (last if last.tzinfo is not None else last.replace(tzinfo=timezone.utc))
if age.total_seconds() < 24 * 3600:
yield f"Skipping job {job_url} (fetched {age.seconds//3600}h ago)...\n"
return None
try: try:
job_id = url_to_job_id(job_url) job_id = url_to_job_id(job_url)
yield f"Fetching job page: {job_url}\n" yield f"Fetching job page: {job_url}\n"

2
web/db.py
View File

@@ -14,10 +14,8 @@ Tables:
""" """
from datetime import datetime, UTC from datetime import datetime, UTC
import os
from typing import Optional, Dict, Any, List from typing import Optional, Dict, Any, List
from web.utils import ( from web.utils import (
get_url_from_filename,
get_color_from_string, get_color_from_string,
url_to_job_id, url_to_job_id,
normalize_job_id, normalize_job_id,