Fix tokenizer compatibility issues

app.py

@@ -9,22 +9,45 @@ def load_model():
     """Load the intent classification model"""
     global _classifier
     if _classifier is None:
-        try:
-            # Try loading with trust_remote_code for better compatibility
-            _classifier = pipeline(
-                "text-classification",
-                model="YosefA/adfluence-intent-model",
-                return_all_scores=True,
-                trust_remote_code=True
-            )
-            print("Model loaded successfully!")
-        except Exception as e:
-            print(f"Error loading model YosefA/adfluence-intent-model: {e}")
-            print("Please verify that:")
-            print("1. The model name is correct")
-            print("2. The model exists and is public")
-            print("3. You have internet connection")
-            return None
+        model_name = "YosefA/adfluence-intent-model"
+        
+        # Try multiple approaches to load the model
+        loading_strategies = [
+            {
+                "name": "Standard loading with trust_remote_code",
+                "kwargs": {"trust_remote_code": True, "return_all_scores": True}
+            },
+            {
+                "name": "Loading with revision='main'",
+                "kwargs": {"revision": "main", "return_all_scores": True}
+            },
+            {
+                "name": "Loading with use_fast=False",
+                "kwargs": {"use_fast": False, "return_all_scores": True}
+            },
+            {
+                "name": "Loading with legacy tokenizer",
+                "kwargs": {"use_fast": False, "trust_remote_code": True, "return_all_scores": True}
+            }
+        ]
+        
+        for strategy in loading_strategies:
+            try:
+                print(f"Trying: {strategy['name']}")
+                _classifier = pipeline(
+                    "text-classification",
+                    model=model_name,
+                    **strategy['kwargs']
+                )
+                print(f"✅ Model loaded successfully using: {strategy['name']}")
+                return _classifier
+                
+            except Exception as e:
+                print(f"❌ Failed with {strategy['name']}: {e}")
+                continue
+        
+        print("❌ All loading strategies failed")
+        return None
     return _classifier
 
 def classify_intent(comment):

requirements.txt

@@ -1,7 +1,7 @@
 gradio==4.44.0
-transformers==4.36.0
+transformers==4.45.0
 torch==2.1.0
-tokenizers==0.15.0
-huggingface_hub==0.19.4
+tokenizers==0.20.0
+huggingface_hub==0.25.0
 numpy<2.0.0
 requests==2.31.0