utils/check_config_docstrings.py

# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import importlib
import inspect
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
PATH_TO_TRANSFORMERS = "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
spec = importlib.util.spec_from_file_location(
    "transformers",
    os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
    submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
transformers = spec.loader.load_module()

CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
_re_checkpoint = re.compile("\[(.+?)\]\((https://huggingface\.co/.+?)\)")


CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK = {
    "CLIPConfigMixin",
    "DecisionTransformerConfigMixin",
    "EncoderDecoderConfigMixin",
    "RagConfigMixin",
    "SpeechEncoderDecoderConfigMixin",
    "VisionEncoderDecoderConfigMixin",
    "VisionTextDualEncoderConfigMixin",
}


def check_config_docstrings_have_checkpoints():
    configs_without_checkpoint = []

    for config_class in list(CONFIG_MAPPING.values()):
        checkpoint_found = False

        # source code of `config_class`
        config_source = inspect.getsource(config_class)
        checkpoints = _re_checkpoint.findall(config_source)

        for checkpoint in checkpoints:
            # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
            # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
            ckpt_name, ckpt_link = checkpoint

            # verify the checkpoint name corresponds to the checkpoint link
            ckpt_link_from_name = f"https://huggingface.co/{ckpt_name}"
            if ckpt_link == ckpt_link_from_name:
                checkpoint_found = True
                break

        name = config_class.__name__
        if not checkpoint_found and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
            configs_without_checkpoint.append(name)

    if len(configs_without_checkpoint) > 0:
        message = "\n".join(sorted(configs_without_checkpoint))
        raise ValueError(f"The following configurations don't contain any valid checkpoint:\n{message}")


if __name__ == "__main__":
    check_config_docstrings_have_checkpoints()