fetchItchIo: init (#470088)

doc/build-helpers/fetchers.chapter.md

@@ -1003,3 +1003,27 @@ fetchtorrent {
 
 - `config`: When using `transmission` as the `backend`, a json configuration can
   be supplied to transmission. Refer to the [upstream documentation](https://github.com/transmission/transmission/blob/main/docs/Editing-Configuration-Files.md) for information on how to configure.
+
+## `fetchItchIo` {#fetchitchio}
+
+`fetchItchIo` is a fetcher for downloading game assets from [itch.io](https://itch.io/). It accepts these arguments:
+
+- `gameUrl`: The store page URL of the game.
+- `upload`: The numerical ID of the asset to download. To find the upload ID of an asset, check the basename of the request URL when you download the asset using a browser.
+- `hash`.
+- `name` (optional): The derivation name, often the filename of the asset.
+- `extraMessage` (optional): Extra message printed if the API key is not provided or if the account did not purchase the game.
+
+For this fetcher to work, the environment variable `NIX_ITCHIO_API_KEY` must be set for the nix building process (which is nix-daemon in multi-user mode), and it must belong to an account that has bought the game if it is behind a paywall.
+To get your API key, go to the ["API key" section](https://itch.io/user/settings/api-keys) of your account settings on itch.io.
+
+```nix
+{ fetchItchIo }:
+
+fetchItchIo {
+  name = "DungeonDuelMonsters-linux-x64.zip";
+  hash = "sha256-gq2nGwpaStqaVI1pL63xygxOI/z53o+zLwiKizG98Ks=";
+  gameUrl = "https://mikaygo.itch.io/ddm";
+  upload = "13371354";
+}
+```

doc/redirects.json

@@ -1818,6 +1818,9 @@
   "fetchtorrent-parameters": [
     "index.html#fetchtorrent-parameters"
   ],
+  "fetchitchio": [
+    "index.html#fetchitchio"
+  ],
   "chap-trivial-builders": [
     "index.html#chap-trivial-builders"
   ],

pkgs/build-support/fetchitchio/default.nix

@@ -0,0 +1,115 @@
+{
+  lib,
+  stdenvNoCC,
+  python3,
+}:
+
+lib.extendMkDerivation {
+  constructDrv = stdenvNoCC.mkDerivation;
+  excludeDrvArgNames = [
+    "derivationArgs"
+    "sha1"
+    "sha256"
+    "sha512"
+  ];
+  extendDrvArgs =
+    finalAttrs:
+    lib.fetchers.withNormalizedHash { } (
+      {
+
+        endpoint ? "https://api.itch.io",
+
+        # The name of the environment variable that contains the itch.io API key.
+        # The environment variable needs to be set for the nix building process,
+        # which is nix-daemon for multi-user mode.
+        apiKeyVar ? "NIX_ITCHIO_API_KEY",
+
+        # The game store page URL in the format of https://{author}.itch.io/{game}
+        gameUrl,
+
+        # The upload ID of the downloadable file.
+        # To get the upload ID, look at the request URL when you download it.
+        upload,
+
+        # Derivation name.
+        name ? null,
+
+        # The extra message printed when the API key is not provided
+        # or when the account of the API key did not purchase the game.
+        extraMessage ? null,
+
+        # Show the download URL without actually downloading it, for testing purposes.
+        # Notice that this can potentially leak the API key.
+        showUrl ? false,
+
+        outputHash ? lib.fakeHash,
+        outputHashAlgo ? null,
+        preFetch ? "",
+        postFetch ? "",
+        nativeBuildInputs ? [ ],
+        impureEnvVars ? [ ],
+        passthru ? { },
+        meta ? { },
+        preferLocalBuild ? true,
+        derivationArgs ? { },
+      }:
+      let
+        finalHashHasColon = lib.hasInfix ":" finalAttrs.hash;
+        finalHashColonMatch = lib.match "([^:]+)[:](.*)" finalAttrs.hash;
+      in
+      derivationArgs
+      // {
+        __structuredAttrs = true;
+
+        name = if name != null then name else baseNameOf gameUrl;
+
+        hash =
+          if outputHashAlgo == null || outputHash == "" || lib.hasPrefix outputHashAlgo outputHash then
+            outputHash
+          else
+            "${outputHashAlgo}:${outputHash}";
+        outputHash =
+          if finalAttrs.hash == "" then
+            lib.fakeHash
+          else if finalHashHasColon then
+            lib.elemAt finalHashColonMatch 1
+          else
+            finalAttrs.hash;
+        outputHashAlgo = if finalHashHasColon then lib.head finalHashColonMatch else null;
+        outputHashMode = "flat";
+
+        nativeBuildInputs = [ python3 ] ++ nativeBuildInputs;
+
+        inherit preferLocalBuild;
+
+        # ENV
+        nixpkgsVersion = lib.trivial.release;
+        uploadName = name;
+        inherit
+          endpoint
+          apiKeyVar
+          gameUrl
+          extraMessage
+          showUrl
+          preFetch
+          postFetch
+          ;
+        impureEnvVars =
+          lib.fetchers.proxyImpureEnvVars
+          ++ [
+            apiKeyVar
+            "NIX_CONNECT_TIMEOUT"
+          ]
+          ++ impureEnvVars;
+
+        builder = builtins.toFile "builder.sh" ''
+          source "$NIX_ATTRS_SH_FILE"
+          runHook preFetch
+          python ${./fetchitchio.py}
+          runHook postFetch
+        '';
+      }
+    );
+
+  inheritFunctionArgs = false;
+}

pkgs/build-support/fetchitchio/fetchitchio.py

@@ -0,0 +1,76 @@
+import itertools
+import json
+import os
+import platform
+import shutil
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+with open(os.environ['NIX_ATTRS_JSON_FILE']) as env_file:
+    ENV = json.load(env_file)
+
+USER_AGENT = f'Python/{platform.python_version()} Nixpkgs/{ENV['nixpkgsVersion']}'
+TIMEOUT = float(os.environ.get('NIX_CONNECT_TIMEOUT') or '15')
+
+ENDPOINT = ENV['endpoint']
+GAME_URL = ENV['gameUrl']
+UPLOAD_ID = ENV['upload']
+
+def abort(message):
+    if 'extraMessage' in ENV:
+        message = f'{message} {ENV['extraMessage']}'
+    print(message, file=sys.stderr)
+    sys.exit(1)
+
+try:
+    API_KEY = os.environ[ENV['apiKeyVar']]
+except KeyError:
+    abort(
+        f'Either set {ENV['apiKeyVar']} for the nix building process '
+        f'or manually download {ENV.get('uploadName', 'the required file')} '
+        f'from {GAME_URL} and add it to nix store.'
+    )
+
+def urlopen(url_or_request):
+    return urllib.request.urlopen(url_or_request, timeout=TIMEOUT)
+
+with urlopen(f'{GAME_URL}/data.json') as response:
+    data = json.load(response)
+    GAME_ID = data['id']
+    IS_FREE = 'price' not in data
+
+def api(path, params={}, download=False):
+    url = f'{ENDPOINT}{path}?{urllib.parse.urlencode({'api_key': API_KEY, **params})}'
+    if download and ENV['showUrl']:
+        with open(os.environ['out'], 'w') as output_file:
+            print(url, file=output_file)
+        return
+    request = urllib.request.Request(url, headers={'User-Agent': USER_AGENT})
+    with urlopen(request) as response:
+        if download:
+            with open(os.environ['out'], 'wb') as output_file:
+                shutil.copyfileobj(response, output_file)
+        else:
+            return json.load(response)
+
+if IS_FREE:
+    api(f'/uploads/{UPLOAD_ID}/download', download=True)
+    sys.exit()
+
+KEY_ID = None
+for page in itertools.count(1):
+    data = api('/profile/owned-keys', {'page': page})
+    if 'owned_keys' not in data:
+        break
+    for key in data['owned_keys']:
+        if key['game_id'] == GAME_ID:
+            KEY_ID = key['id']
+            break
+    if len(data['owned_keys']) < data['per_page']:
+        break
+if not KEY_ID:
+    abort(f'Cannot find a key associated with {GAME_URL}. Did you buy the game?')
+
+api(f'/uploads/{UPLOAD_ID}/download', {'download_key_id': KEY_ID}, download=True)

pkgs/by-name/ce/celestegame/celeste.nix

@@ -57,6 +57,7 @@ stdenvNoCC.mkDerivation {
 
   src =
     if overrideSrc == null then
+      # TODO: Replace this with fetchItchIo
       requireFile {
         name = "celeste-linux.zip";
         hash = "sha256-phNDBBHb7zwMRaBHT5D0hFEilkx9F31p6IllvLhHQb8=";

pkgs/by-name/dd/ddm/package.nix

@@ -1,7 +1,7 @@
 {
   stdenvNoCC,
   lib,
-  requireFile,
+  fetchItchIo,
   asar,
   copyDesktopItems,
   electron,
@@ -18,10 +18,11 @@ stdenvNoCC.mkDerivation (finalAttrs: {
   pname = "ddm";
   version = "4.1.0";
 
-  src = requireFile {
+  src = fetchItchIo {
     name = "DungeonDuelMonsters-linux-x64.zip";
     hash = "sha256-gq2nGwpaStqaVI1pL63xygxOI/z53o+zLwiKizG98Ks=";
-    url = "https://mikaygo.itch.io/ddm";
+    gameUrl = "https://mikaygo.itch.io/ddm";
+    upload = "13371354";
   };
 
   strictDeps = true;

pkgs/by-name/ko/koboredux/package.nix

@@ -31,6 +31,7 @@ let
     sha256 = "09h9r65z8bar2z89s09j6px0gdq355kjf38rmd85xb2aqwnm6xig";
   };
 
+  # TODO: Replace this with fetchItchIo
   assets_src = requireFile {
     name = "koboredux-${version}-Linux.tar.bz2";
     sha256 = "11bmicx9i11m4c3dp19jsql0zy4rjf5a28x4hd2wl8h3bf8cdgav";

pkgs/top-level/all-packages.nix

@@ -685,6 +685,8 @@ with pkgs;
 
   fetchgx = callPackage ../build-support/fetchgx { };
 
+  fetchItchIo = callPackage ../build-support/fetchitchio { };
+
   fetchPypi = callPackage ../build-support/fetchpypi { };
 
   fetchPypiLegacy = callPackage ../build-support/fetchpypilegacy { };