scripts/beir/gather_beir_index_stats.py

#
# Pyserini: Reproducible IR research with sparse and dense representations
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os

from pyserini.index.lucene import IndexReader
from pyserini.util import compute_md5

beir_keys = {
    'trec-covid': 'TREC-COVID',
    'bioasq': 'BioASQ',
    'nfcorpus': 'NFCorpus',
    'nq': 'NQ',
    'hotpotqa': 'HotpotQA',
    'fiqa': 'FiQA-2018',
    'signal1m': 'Signal-1M',
    'trec-news': 'TREC-NEWS',
    'robust04': 'Robust04',
    'arguana': 'ArguAna',
    'webis-touche2020': 'Webis-Touche2020',
    'cqadupstack-android': 'CQADupStack-android',
    'cqadupstack-english': 'CQADupStack-english',
    'cqadupstack-gaming': 'CQADupStack-gaming',
    'cqadupstack-gis': 'CQADupStack-gis',
    'cqadupstack-mathematica': 'CQADupStack-mathematica',
    'cqadupstack-physics': 'CQADupStack-physics',
    'cqadupstack-programmers': 'CQADupStack-programmers',
    'cqadupstack-stats': 'CQADupStack-stats',
    'cqadupstack-tex': 'CQADupStack-tex',
    'cqadupstack-unix': 'CQADupStack-unix',
    'cqadupstack-webmasters': 'CQADupStack-webmasters',
    'cqadupstack-wordpress': 'CQADupStack-wordpress',
    'quora': 'Quora',
    'dbpedia-entity': 'DBPedia',
    'scidocs': 'SCIDOCS',
    'fever': 'FEVER',
    'climate-fever': 'Climate-FEVER',
    'scifact': 'SciFact'
}

commitid = '505594'
date = '20221116'
type = 'flat'

for key in beir_keys:
    index_reader = IndexReader(f'indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}')
    stats = index_reader.stats()
    md5 = compute_md5(f'indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz')
    size = os.path.getsize(f'indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz')
    print(f'    "beir-v1.0.0-{key}.{type}": {{')
    print(f'        "description": "Lucene flat index of BEIR (v1.0.0): {beir_keys[key]}",')
    print(f'        "filename": "lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz",')
    print(f'        "readme": "lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.README.md",')
    print(f'        "urls": [')
    print(f'            "https://rgw.cs.uwaterloo.ca/pyserini/indexes/lucene-index.beir-v1.0.0-{key}.{type}.{date}.{commitid}.tar.gz"')
    print(f'        ],')
    print(f'        "md5": "{md5}",')
    print(f'        "size compressed (bytes)": {size},')
    print(f'        "total_terms": {stats["total_terms"]},')
    print(f'        "documents": {stats["documents"]},')
    print(f'        "unique_terms": {stats["unique_terms"]},')
    print(f'        "downloaded": False')
    print(f'    }},')

# Stats for "contriever" indexes
for key in beir_keys:
    index_reader = IndexReader(f'indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}')
    stats = index_reader.stats()
    md5 = compute_md5(f'indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz')
    size = os.path.getsize(f'indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz')
    print(f'    "beir-v1.0.0-{key}.contriever": {{')
    print(f'        "description": "Faiss index for BEIR v1.0.0 ({beir_keys[key]}) corpus encoded by Contriever encoder.",')
    print(f'        "filename": "faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz",')
    print(f'        "readme": "faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.README.md",')
    print(f'        "urls": [')
    print(f'            "https://rgw.cs.uwaterloo.ca/JIMMYLIN-bucket0/pyserini-indexes/faiss.beir-v1.0.0-{key}.contriever.{date}.{commitid}.tar.gz"')
    print(f'        ],')
    print(f'        "md5": "{md5}",')
    print(f'        "size compressed (bytes)": {size},')
    print(f'        "documents": {stats["documents"]},')
    print(f'        "downloaded": False,')
    print(f'        "texts": "beir-v1.0.0-{key}.flat"')
    print(f'    }},')

# Stats for "contriever" indexes with msmarco-ft
for key in beir_keys:
    index_reader = IndexReader(f'indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}')
    stats = index_reader.stats()
    md5 = compute_md5(f'indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.tar.gz')
    size = os.path.getsize(f'indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.tar.gz')
    print(f'    "beir-v1.0.0-{key}.contriever": {{')
    print(f'        "description": "Faiss index for BEIR v1.0.0 ({beir_keys[key]}) corpus encoded by Contriever encoder fine-tuned with MS MARCO.",')
    print(f'        "filename": "faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.tar.gz",')
    print(f'        "readme": "faiss.beir-v1.0.0-{key}.contriever-msmarco.{date}.README.md",')
    print(f'        "urls": [')
    print(f'            "https://rgw.cs.uwaterloo.ca/pyserini/indexes/faiss.beir-v1.0.0-{key}.contriever-msmarco.20230124.tar.gz"')
    print(f'        ],')
    print(f'        "md5": "{md5}",')
    print(f'        "size compressed (bytes)": {size},')
    print(f'        "documents": {stats["documents"]},')
    print(f'        "downloaded": False,')
    print(f'        "texts": "beir-v1.0.0-{key}.flat"')
    print(f'    }},')