remove caching

web/app.py

@@ -1,5 +1,5 @@
 import os
-from flask import Flask, request, jsonify, render_template, redirect, url_for, session, flash, send_file
+from flask import Flask, request, jsonify, render_template, redirect, url_for, session, flash, Response
 from flask_wtf import CSRFProtect
 from typing import Dict, List
 
@@ -34,7 +34,6 @@ from web.utils import (
     initialize_users_from_settings,
     filter_jobs,
     get_job_by_id,
-    get_cache_dir,
 )
 from web.db import get_all_regions, get_all_keywords
 
@@ -232,39 +231,6 @@ def job_by_id(job_id):
     return jsonify({"error": "Job not found"}), 404
 
 
-@app.route('/cached/<job_id>', methods=['GET'])
-def serve_cached(job_id):
-    """Serve the cached HTML file for a job if available.
-
-    Uses the job record's `file_path_abs` when present, or resolves the DB `file_path` via helper.
-    Ensures the returned file is located under the configured cache directory to avoid path-traversal.
-    """
-    try:
-        from web.db import db_get_cached_abs_path
-        j = get_job_by_id(job_id)
-        if not j:
-            return "Job not found", 404
-
-        # Prefer file_path_abs, fall back to resolving the DB-stored file_path
-        abs_fp = j.get('file_path_abs') or None
-        if not abs_fp:
-            db_fp = j.get('file_path')
-            abs_fp = db_get_cached_abs_path(db_fp) if db_fp else None
-
-        if not abs_fp or not os.path.isfile(abs_fp):
-            return "Cached file not available", 404
-
-        cache_dir = os.path.abspath(get_cache_dir())
-        abs_fp = os.path.abspath(abs_fp)
-        # Ensure the file is inside the cache directory
-        if os.path.commonpath([cache_dir, abs_fp]) != cache_dir:
-            return "Forbidden", 403
-
-        return send_file(abs_fp)
-    except Exception:
-        return "Error serving cached file", 500
-
-
 @app.route('/jobs/<job_id>/favorite', methods=['POST'])
 def set_favorite(job_id):
     """Mark or unmark a job as favorite for a given user.
@@ -290,9 +256,21 @@ csrf.exempt(set_favorite)
 
 @app.route('/scrape', methods=['GET'])
 def scrape():
-    """Trigger the web scraping process."""
-    scraper()
-    return jsonify({"status": "Scraping completed"})
+    """Trigger the web scraping process with streaming output."""
+    def generate():
+        try:
+            for message in scraper():
+                yield message
+        except Exception as e:
+            yield f"Error during scraping: {str(e)}\n"
+
+    return Response(generate(), mimetype='text/plain')
+
+
+@app.route('/scrape-page', methods=['GET'])
+def scrape_page():
+    """Serve the scrape page with streaming output display."""
+    return render_template('scrape.html', title='Scrape Jobs')
 
 
 # ---------------- Auth & Admin UI ------------------------------------------