nixpkgs/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py

#!/usr/bin/env nix-shell
#!nix-shell ./update-shell.nix -i python

import json
import logging
import os
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import requests

log = logging.getLogger("vim-updater")

NURR_JSON_URL = (
    "https://raw.githubusercontent.com/lumen-oss/nurr/main/tree-sitter-parsers.json"
)
NVIM_TREESITTER_QUERIES_URL = "https://api.github.com/repos/nvim-treesitter/nvim-treesitter/contents/runtime/queries"


def generate_grammar(lang, parser_info, parsers_map):
    """Generate grammar for a language based on the parser info"""
    if "install_info" not in parser_info:
        log.warning(f"Parser {lang} does not have install_info, skipping")
        return ""

    install_info = parser_info["install_info"]

    url = install_info["url"]
    rev = install_info["revision"]

    generated = f"""    {lang} = buildGrammar {{
      language = "{lang}";
      version = "0.0.0+rev={rev[:7]}";
      src = """

    generated += subprocess.check_output(["nurl", url, rev, "--indent=6"], text=True)
    generated += ";"

    location = install_info.get("location", "")
    if location:
        generated += f"""
      location = "{location}";"""

    if install_info.get("generate", False):
        generated += """
      generate = true;"""

    # Add requires field - only include parsers
    requires = parser_info.get("requires", [])
    if requires:
        # Filter to only include parser dependencies (those with install_info)
        parser_requires = [
            req
            for req in requires
            if req in parsers_map and "install_info" in parsers_map[req]
        ]
        if parser_requires:
            generated += """
      passthru.requires = [
"""
            for req in parser_requires:
                generated += f'        "{req}"\n'
            generated += "      ];"

    generated += f"""
      meta.homepage = "{url}";
    }};
"""

    return generated


def generate_query(lang: str, parser_info: dict | None, queries_set: set[str]):
    """Generate query derivation for a language"""
    generated = f"""    {lang} = buildQueries {{
      language = "{lang}";"""

    # Add requires field for queries - include everything that has queries
    if parser_info and "requires" in parser_info:
        requires = parser_info["requires"]
        # Filter to only include langs that have queries
        query_requires = [req for req in requires if queries_set and req in queries_set]
        if query_requires:
            generated += """
      requires = [
"""
            for req in query_requires:
                generated += f'        "{req}"\n'
            generated += "      ];"

    generated += """
    };
"""
    return generated


def fetch_nurr_parsers():
    """Fetch the parser information from nurr repository"""
    log.info("Fetching parser data from %s", NURR_JSON_URL)

    headers = {}
    github_token = os.environ.get("GITHUB_TOKEN")
    if github_token:
        log.info("Using GITHUB_TOKEN for authentication")
        headers["Authorization"] = f"token {github_token}"
    else:
        log.warning("No GITHUB_TOKEN found. GitHub API requests may be rate-limited.")

    response = requests.get(NURR_JSON_URL, headers=headers, timeout=30)
    response.raise_for_status()
    data = response.json()

    try:
        parsers = data["parsers"]
    except KeyError:
        raise ValueError(
            "Unexpected response from NURR:\n" + json.dumps(data, indent=2)
        )
    log.info(f"Successfully fetched {len(parsers)} parsers")
    return parsers


def fetch_available_queries():
    """Fetch list of languages that have queries in nvim-treesitter repo"""
    log.info("Fetching available queries from %s", NVIM_TREESITTER_QUERIES_URL)

    headers = {}
    github_token = os.environ.get("GITHUB_TOKEN")
    if github_token:
        headers["Authorization"] = f"token {github_token}"

    response = requests.get(NVIM_TREESITTER_QUERIES_URL, headers=headers, timeout=30)
    response.raise_for_status()
    data = response.json()

    # We assume items with type="dir" in the queries/ folder are language names
    languages = sorted([item["name"] for item in data if item["type"] == "dir"])
    log.info(f"Found {len(languages)} languages with queries")
    return languages


def process_parser_info(parser_info, parsers_map):
    """Process a single parser info entry and generate grammar for it"""
    return generate_grammar(parser_info["lang"], parser_info, parsers_map)


def update_grammars():
    """Update grammar definitions using nurr's parser information"""
    parsers_info = fetch_nurr_parsers()
    queries_list = fetch_available_queries()

    # Create a mapping of lang -> parser_info for quick lookup
    parsers_map = {p["lang"]: p for p in parsers_info}

    generated_file = """# generated by pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py
# Using parser data from https://github.com/nvim-neorocks/nurr/blob/main/tree-sitter-parsers.json

{
  buildGrammar,
  buildQueries,
  """

    nurl_output = subprocess.check_output(["nurl", "-Ls", ","], text=True).strip()
    indented_output = nurl_output.replace(",", ",\n  ")
    generated_file += indented_output
    generated_file += """,
}:

{
  parsers = {
"""

    # Process parsers in parallel for better performance
    with ThreadPoolExecutor(max_workers=5) as executor:
        for generated in executor.map(
            lambda p: process_parser_info(p, parsers_map), parsers_info
        ):
            generated_file += generated

    generated_file += """  };

  queries = {
"""

    # Convert queries list to a set for fast lookup
    queries_set = set(queries_list)

    # Process queries - include parser info if available for requires field
    for lang in queries_list:
        parser_info = parsers_map.get(lang)
        generated_file += generate_query(lang, parser_info, queries_set)

    generated_file += "  };\n}\n"
    return generated_file


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

    generated = update_grammars()
    output_path = Path(__file__).parent.parent.parent / "nvim-treesitter/generated.nix"
    log.info("Writing output to %s", output_path)
    with open(output_path, "w") as f:
        f.write(generated)
    log.info("Successfully updated grammar definitions")