add exact match metadata

app.py

@@ -3,6 +3,8 @@ import os
 import gradio as gr
 import requests
 from huggingface_hub import HfApi
+import traceback
+
 
 hf_api = HfApi()
 roots_datasets = {
@@ -52,11 +54,32 @@ def process_pii(text):
     return text
 
 
+def format_meta(result):
+    meta_html = (
+        """
+              <p class='underline-on-hover' style='font-size:12px; font-family: Arial; color:#585858; text-align: left;'>
+              <a href='{}' target='_blank'>{}</a></p>""".format(
+            result["meta"]["url"], result["meta"]["url"]
+        )
+        if "meta" in result and result["meta"] is not None and "url" in result["meta"]
+        else ""
+    )
+    docid_html = get_docid_html(result["docid"])
+    return """{}
+          <p style='font-size:14px; font-family: Arial; color:#7978FF; text-align: left;'>Document ID: {}</p>
+          <p style='font-size:12px; font-family: Arial; color:MediumAquaMarine'>Language: {}</p>
+      """.format(
+        meta_html,
+        docid_html,
+        result["lang"] if lang in result else None,
+    )
+    return meta_html
+
+
 def process_results(results, highlight_terms):
     if len(results) == 0:
         return """<br><p style='font-family: Arial; color:Silver; text-align: center;'>
                 No results retrieved.</p><br><hr>"""
-
     results_html = ""
     for result in results:
         tokens = result["text"].split()
@@ -68,136 +91,102 @@ def process_results(results, highlight_terms):
                 tokens_html.append(token)
         tokens_html = " ".join(tokens_html)
         tokens_html = process_pii(tokens_html)
-        meta_html = (
-            """
-                <p class='underline-on-hover' style='font-size:12px; font-family: Arial; color:#585858; text-align: left;'>
-                <a href='{}' target='_blank'>{}</a></p>""".format(
-                result["meta"]["url"], result["meta"]["url"]
-            )
-            if "meta" in result
-            and result["meta"] is not None
-            and "url" in result["meta"]
-            else ""
-        )
-        docid_html = get_docid_html(result["docid"])
-        results_html += """{}
-            <p style='font-size:14px; font-family: Arial; color:#7978FF; text-align: left;'>Document ID: {}</p>
-            <p style='font-size:12px; font-family: Arial; color:MediumAquaMarine'>Language: {}</p>
+        meta_html = format_meta(result)
+        meta_html += """
             <p style='font-family: Arial;'>{}</p>
             <br>
         """.format(
-            meta_html, docid_html, result["lang"], tokens_html
+            tokens_html
         )
+        results_html += meta_html
     return results_html + "<hr>"
 
 
-def scisearch(query, language, num_results=10, exact_search=False):
-    try:
-        query = " ".join(query.split())
-        if query == "" or query is None:
-            return ""
-
-        post_data = {"query": query, "k": num_results, "exact_search": exact_search}
-        if language != "detect_language":
-            post_data["lang"] = language
-
-        output = requests.post(
-            os.environ.get("address"),
-            headers={"Content-type": "application/json"},
-            data=json.dumps(post_data),
-            timeout=60,
-        )
+def process_exact_match_payload(payload, query):
+    results = payload["results"]
+    results_html = ""
+    for result in results:
+        text = result["text"]
+        print(result, text, type(text))
+        meta_html = format_meta(result)
+        result_html = """<br><hr><br>""" + meta_html
+        query_start = text.find(query)
+        query_end = query_start + len(query)
+        result_html += text[0:query_start]
+        result_html += "<b>{}</b>".format(text[query_start:query_end])
+        result_html += text[query_end:]
+        results_html += result_html
+    return results_html + "<hr>"
 
-        payload = json.loads(output.text)
 
-        if "err" in payload:
-            if payload["err"]["type"] == "unsupported_lang":
-                detected_lang = payload["err"]["meta"]["detected_lang"]
-                return f"""
-                    <p style='font-size:18px; font-family: Arial; color:MediumVioletRed; text-align: center;'>
-                    Detected language <b>{detected_lang}</b> is not supported.<br>
-                    Please choose a language from the dropdown or type another query.
-                    </p><br><hr><br>"""
-
-        results = payload["results"]
-        highlight_terms = payload["highlight_terms"]
-
-        if language == "detect_language":
-            return (
-                (
-                    f"""<p style='font-family: Arial; color:MediumAquaMarine; text-align: center; line-height: 3em'>
-                Detected language: <b>{results[0]["lang"]}</b></p><br><hr><br>"""
-                    if len(results) > 0 and language == "detect_language"
-                    else ""
-                )
-                + process_results(results, highlight_terms)
+def process_bm25_match_payload(payload, language):
+    if "err" in payload:
+        if payload["err"]["type"] == "unsupported_lang":
+            detected_lang = payload["err"]["meta"]["detected_lang"]
+            return f"""
+                <p style='font-size:18px; font-family: Arial; color:MediumVioletRed; text-align: center;'>
+                Detected language <b>{detected_lang}</b> is not supported.<br>
+                Please choose a language from the dropdown or type another query.
+                </p><br><hr><br>"""
+
+    results = payload["results"]
+    highlight_terms = payload["highlight_terms"]
+
+    if language == "detect_language":
+        return (
+            (
+                f"""<p style='font-family: Arial; color:MediumAquaMarine; text-align: center; line-height: 3em'>
+            Detected language: <b>{results[0]["lang"]}</b></p><br><hr><br>"""
+                if len(results) > 0 and language == "detect_language"
+                else ""
             )
+            + process_results(results, highlight_terms)
+        )
 
-        if language == "all":
-            results_html = ""
-            for lang, results_for_lang in results.items():
-                if len(results_for_lang) == 0:
-                    results_html += f"""<p style='font-family: Arial; color:Silver; text-align: left; line-height: 3em'>
-                            No results for language: <b>{lang}</b><hr></p>"""
-                    continue
-
-                collapsible_results = f"""
-                    <details>
-                        <summary style='font-family: Arial; color:MediumAquaMarine; text-align: left; line-height: 3em'>
-                            Results for language: <b>{lang}</b><hr>
-                        </summary>
-                        {process_results(results_for_lang, highlight_terms)}
-                    </details>"""
-                results_html += collapsible_results
-            return results_html
-
-        return process_results(results, highlight_terms)
+    if language == "all":
+        results_html = ""
+        for lang, results_for_lang in results.items():
+            if len(results_for_lang) == 0:
+                results_html += f"""<p style='font-family: Arial; color:Silver; text-align: left; line-height: 3em'>
+                        No results for language: <b>{lang}</b><hr></p>"""
+                continue
 
-    except Exception as e:
-        results_html = f"""
-                <p style='font-size:18px; font-family: Arial; color:MediumVioletRed; text-align: center;'>
-                Raised {type(e).__name__}</p>
-                <p style='font-size:14px; font-family: Arial; '>
-                Check if a relevant discussion already exists in the Community tab. If not, please open a discussion.
-                </p>
-            """
+            collapsible_results = f"""
+                <details>
+                    <summary style='font-family: Arial; color:MediumAquaMarine; text-align: left; line-height: 3em'>
+                        Results for language: <b>{lang}</b><hr>
+                    </summary>
+                    {process_results(results_for_lang, highlight_terms)}
+                </details>"""
+            results_html += collapsible_results
+        return results_html
 
-    return results_html
+    return process_results(results, highlight_terms)
 
 
-def perform_exact_search(query, num_results=10):
+def scisearch(query, language, num_results=10, exact_search=False):
     try:
-        print("perform_exact_search")
         query = " ".join(query.split())
         if query == "" or query is None:
             return ""
-
-        post_data = {"query": query, "k": num_results, "exact_search": True}
-
-        print("post_data", post_data)
-
+        post_data = {"query": query, "k": num_results}
+        if language != "detect_language":
+            post_data["lang"] = language
+        address = (
+            "http://34.105.160.81:8080" if exact_search else os.environ.get("address")
+        )
         output = requests.post(
-            "http://34.105.160.81:8080",
+            address,
             headers={"Content-type": "application/json"},
             data=json.dumps(post_data),
             timeout=60,
         )
-
         payload = json.loads(output.text)
-        results = payload["results"]
-
-        results_html = ""
-        for result in results:
-            print(result)
-            result_html = """<br><hr><br>"""
-            query_start = result.find(query)
-            query_end = query_start + len(query)
-            result_html += result[0:query_start]
-            result_html += "<b>{}</b>".format(result[query_start:query_end])
-            result_html += result[query_end:]
-            results_html += result_html
-        return results_html + "<hr>"
-
+        return (
+            process_bm25_match_payload(payload, language)
+            if not exact_search
+            else process_exact_match_payload(payload, query)
+        )
     except Exception as e:
         results_html = f"""
                 <p style='font-size:18px; font-family: Arial; color:MediumVioletRed; text-align: center;'>
@@ -206,6 +195,9 @@ def perform_exact_search(query, num_results=10):
                 Check if a relevant discussion already exists in the Community tab. If not, please open a discussion.
                 </p>
             """
+        print(e)
+        print(traceback.format_exc())
+    return results_html
 
 
 def flag(query, language, num_results, issue_description):
@@ -308,12 +300,6 @@ if __name__ == "__main__":
             query = query.strip()
             if query is None or query == "":
                 return "", ""
-
-            if exact_search:
-                return {
-                    results: perform_exact_search(query, k),
-                    flagging_form: gr.update(visible=True),
-                }
             return {
                 results: scisearch(query, lang, k, exact_search),
                 flagging_form: gr.update(visible=True),