Update app.py

app.py

@@ -2,7 +2,6 @@ import json
 import os
 import re
 import sys
-from functools import lru_cache
 from typing import List, Optional, Tuple, Dict
 
 print("="*60)
@@ -84,14 +83,18 @@ MAX_ROWS = 200
 HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN")
 API = HfApi(token=HF_TOKEN)
 
+# Smart cache: stores PR -> list of file paths
+# Fetches from remote when specific data not found
+_pr_files_cache: Dict[str, List[str]] = {}
 
-@lru_cache(maxsize=128)
-def _list_collection_files(pr_number: str) -> Tuple[str, ...]:
+
+def _fetch_files_for_pr(pr_number: str) -> List[str]:
     """
-    Return the `failure_summary.json` paths stored for a specific PR.
+    Fetch all failure_summary.json files for a given PR from the remote dataset.
+    Returns a list of file paths.
     """
     prefix = f"pr-{pr_number}"
-    print(f"DEBUG: Looking for files with prefix: {prefix}")
+    print(f"DEBUG: Fetching files for PR {pr_number} with prefix: {prefix}")
     print(f"DEBUG: Dataset ID: {DATASET_ID}")
     
     try:
@@ -105,12 +108,12 @@ def _list_collection_files(pr_number: str) -> Tuple[str, ...]:
         print("DEBUG: Successfully called list_repo_tree")
     except HfHubHTTPError as error:
         print(f"ERROR: Failed to list repo tree: {error}")
-        return tuple()
+        return []
     except Exception as error:
         print(f"ERROR: Unexpected error in list_repo_tree: {error}")
         import traceback
         traceback.print_exc()
-        return tuple()
+        return []
 
     files = []
     matching_paths = []
@@ -122,7 +125,7 @@ def _list_collection_files(pr_number: str) -> Tuple[str, ...]:
             entry_type = getattr(entry, "type", type(entry).__name__)
             entry_path = getattr(entry, "path", str(entry))
             
-            # Debug: show all entries
+            # Debug: show first few entries
             if len(all_entries) <= 10:
                 print(f"DEBUG: Entry {len(all_entries)}: {entry_path} (type: {entry_type})")
             
@@ -142,17 +145,68 @@ def _list_collection_files(pr_number: str) -> Tuple[str, ...]:
         
         if matching_paths and len(files) == 0:
             print(f"DEBUG: Sample matching paths (first 5): {matching_paths[:5]}")
-            # Check if we're only getting folders
-            folder_count = sum(1 for p in matching_paths if "RepoFolder" in str(type(p)))
-            print(f"DEBUG: Folders in matching paths: {folder_count}")
             
     except Exception as error:
         print(f"ERROR: Error processing entries: {error}")
         import traceback
         traceback.print_exc()
-        return tuple()
+        return []
+    
+    return files
+
+
+def _extract_commit_from_path(path: str) -> str:
+    """Extract commit SHA from file path."""
+    parts = path.split("/")
+    if len(parts) >= 2 and parts[1].startswith("sha-"):
+        return parts[1][len("sha-") :]
+    return "unknown"
+
+
+def _get_files_for_pr_and_sha(pr_number: str, sha: str = "") -> List[str]:
+    """
+    Get files for a PR, with smart caching.
+    
+    Strategy:
+    1. If PR not in cache, fetch from remote and cache it
+    2. If PR in cache but specific SHA not found, re-fetch and update cache
+    3. Return matching files
+    
+    Args:
+        pr_number: The PR number to query
+        sha: Optional commit SHA to filter for
+    
+    Returns:
+        List of file paths for this PR (optionally filtered by SHA)
+    """
+    sha_lower = sha.lower() if sha else ""
+    
+    # First check: Do we have this PR in cache?
+    if pr_number not in _pr_files_cache:
+        print(f"DEBUG: PR {pr_number} not in cache, fetching from remote...")
+        files = _fetch_files_for_pr(pr_number)
+        _pr_files_cache[pr_number] = files
+        print(f"DEBUG: Cached {len(files)} files for PR {pr_number}")
+    else:
+        files = _pr_files_cache[pr_number]
+        print(f"DEBUG: PR {pr_number} found in cache with {len(files)} files")
+        
+        # Second check: If SHA specified, do we have it in cache?
+        if sha:
+            sha_found = False
+            for file_path in files:
+                commit = _extract_commit_from_path(file_path)
+                if commit.lower().startswith(sha_lower) or sha_lower.startswith(commit.lower()):
+                    sha_found = True
+                    break
+            
+            if not sha_found:
+                print(f"DEBUG: SHA {sha} not found in cache for PR {pr_number}, re-fetching...")
+                files = _fetch_files_for_pr(pr_number)
+                _pr_files_cache[pr_number] = files
+                print(f"DEBUG: Updated cache with {len(files)} files for PR {pr_number}")
     
-    return tuple(files)
+    return files
 
 
 def _load_payload(path: str) -> Optional[dict]:
@@ -174,13 +228,6 @@ def _load_payload(path: str) -> Optional[dict]:
         return None
 
 
-def _extract_commit_from_path(path: str) -> str:
-    parts = path.split("/")
-    if len(parts) >= 2 and parts[1].startswith("sha-"):
-        return parts[1][len("sha-") :]
-    return "unknown"
-
-
 def _filter_records(repo: str, pr: str, sha: str) -> List[dict]:
     repo = repo.strip().lower()
     pr = pr.strip()
@@ -191,7 +238,8 @@ def _filter_records(repo: str, pr: str, sha: str) -> List[dict]:
     if not pr:
         return []
 
-    file_paths = _list_collection_files(pr)
+    # Use smart cache that auto-fetches if needed
+    file_paths = _get_files_for_pr_and_sha(pr, sha)
     print(f"DEBUG: Found {len(file_paths)} file paths")
     
     records: List[dict] = []
@@ -520,7 +568,7 @@ Files are organized as `pr-{{PR}}/sha-{{COMMIT}}/failure_summary.json`
 
 
 def refresh_dataset() -> str:
-    _list_collection_files.cache_clear()
+    _pr_files_cache.clear()
     return "✅ Cleared cached manifest. Data will be reloaded on next search."