Use transifex-python lib with v3 API. Add stat files

Author: kazanzhy

.github/scripts/manage_translation.py

@@ -2,215 +2,101 @@
 #
 # This python file contains utility scripts to manage Python docs Ukrainian translation.
 # It has to be run inside the python-docs-uk git root directory.
-#
-# Inspired by django-docs-translations script by claudep.
-#
-# The following commands are available:
-#
-# * fetch: fetch translations from transifex.com and strip source lines from the
-#          files.
-# * recreate_readme: recreate readme to update translation progress.
-# * regenerate_tx_config: recreate configuration for all resources.
 
-from argparse import ArgumentParser
-from collections import Counter
 import os
-from re import match
-from subprocess import call, run
-import sys
+import re
+from argparse import ArgumentParser
+from pathlib import Path
 
-LANGUAGE = 'uk'
-RESOURCE_NAME_MAP = {'glossary_': 'glossary'}
-TX_ORGANISATION = 'python-doc'
-TX_PROJECT = 'python-newest'
-GH_ORGANISATION = 'python'
-GH_PROJECT = 'python-docs-uk'
-
-def fetch():
-    """
-    Fetch translations from Transifex, remove source lines.
-    """
-    if call("tx --version", shell=True) != 0:
-        sys.stderr.write("The Transifex client app is required (https://developers.transifex.com/docs/cli).\n")
-        exit(1)
-    lang = LANGUAGE
-    pull_returncode = call(f'tx pull -l {lang} --minimum-perc=1 --force --skip', shell=True)
-    if pull_returncode != 0:
-        exit(pull_returncode)
-    for root, _, po_files in os.walk('../..'):
-        for po_file in po_files:
-            if not po_file.endswith(".po"):
-                continue
-            po_path = os.path.join(root, po_file)
-            call(f'msgcat --no-location -o {po_path} {po_path}', shell=True)
-
-
-def recreate_tx_config():
-    """
-    Regenerate Transifex client config for all resources.
-    """
-    resources = _get_resources()
-    with open('.tx/config', 'w') as config:
-        config.writelines(('[main]\n', 'host = https://www.transifex.com\n',))
-        for resource in resources:
-            slug = resource['slug']
-            name = RESOURCE_NAME_MAP.get(slug, slug)
-            if slug == '0':
-                continue
-            elif '--' in slug:
-                directory, file_name = name.split('--')
-                if match(r'\d+_\d+', file_name):
-                    file_name = file_name.replace('_', '.')
-                config.writelines(
-                    (
-                        '\n',
-                        f'[{TX_PROJECT}.{slug}]\n',
-                        f'trans.{LANGUAGE} = {directory}/{file_name}.po\n',
-                        'type = PO\n',
-                        'source_lang = en\n',
-                    )
-                )
-            else:
-                config.writelines(
-                    (
-                        '\n',
-                        f'[{TX_PROJECT}.{slug}]\n',
-                        f'trans.{LANGUAGE} = {name}.po\n',
-                        'type = PO\n',
-                        'source_lang = en\n',
-                    )
-                )
-
-
-def _get_resources():
-    from requests import get
-
-    resources = []
-    offset = 0
-    if os.path.exists('.tx/api-key'):
-        with open('.tx/api-key') as f:
-            transifex_api_key = f.read()
-    else:
-        transifex_api_key = os.getenv('TX_TOKEN')
-    while True:
-        response = get(
-            f'https://api.transifex.com/organizations/{TX_ORGANISATION}/projects/{TX_PROJECT}/resources/',
-            params={'language_code': LANGUAGE, 'offset': offset},
-            auth=('api', transifex_api_key),
-        )
-        response.raise_for_status()
-        response_list = response.json()
-        resources.extend(response_list)
-        if len(response_list) < 100:
-            break
-        offset += len(response_list)
-    return resources
-
-
-def _get_unique_translators():
-    process = run(
-        ['grep', '-ohP', r'(?<=^# )(.+)(?=, \d+$)', '-r', '.'],
-        capture_output=True,
-        text=True,
-    )
-    translators = [match('(.*)( <.*>)?', t).group(1) for t in process.stdout.splitlines()]
-    unique_translators = Counter(translators)
-    return unique_translators
-
-
-def recreate_readme():
-    def language_switcher(entry):
-        return (
-            entry['name'].startswith('bugs')
-            or entry['name'].startswith('tutorial')
-            or entry['name'].startswith('library--functions')
-        )
-
-    def average(averages, weights):
-        return sum([a * w for a, w in zip(averages, weights)]) / sum(weights)
-
-    resources = _get_resources()
-    filtered = list(filter(language_switcher, resources))
-    average_list = [e['stats']['translated']['percentage'] for e in filtered]
-    weights_list = [e['wordcount'] for e in filtered]
-
-    language_switcher_status = average(average_list, weights=weights_list) * 100
-    unique_translators = _get_unique_translators()
-    number_of_translators = len(unique_translators)
-
-    with open('README.md', 'w') as file:
-        file.write(
-            f'''\
-Український переклад документації Python
-========================================
-![build](https://github.com/{GH_ORGANISATION}/{GH_PROJECT}/workflows/.github/workflows/update-and-build.yml/badge.svg)
-![{language_switcher_status:.2f}% прогрес перекладу](https://img.shields.io/badge/прогрес_перекладу-{language_switcher_status:.2f}%25-0.svg)
-![хід перекладу всієї документації](https://img.shields.io/badge/dynamic/json.svg?label=всього&query=$.{LANGUAGE}&url=http://gce.zhsj.me/python/newest)
-![{number_of_translators} перекладачів](https://img.shields.io/badge/перекладачів-{number_of_translators}-0.svg)
-
-Якщо ви знайшли помилку або маєте пропозицію,
-[додати issue](https://github.com/{GH_ORGANISATION}/{GH_PROJECT}/issues) у цьому проекті або запропонуйте зміни:
-
-* Зареєструйтесь на платформі [Transifex](https://www.transifex.com/) 
-* Перейдіть на сторінку [документації Python](https://www.transifex.com/{TX_ORGANISATION}/{TX_PROJECT}/).
-* Натисніть кнопку „Join Team”, оберіть українську (uk) мову та натисніть „Join” щоб приєднатися до команди.
-* Приєднавшись до команди, виберіть ресурс, що хочете виправити/оновити.
-
-Додаткову інформацію про використання Transifex дивіться [в документації](https://docs.transifex.com/getting-started-1/translators).
-
-**Прогрес перекладу**
-
-![прогрес перекладу](language-switcher-progress.svg)
-
-Українська мова з’явиться в меню вибору мови docs.python.org, [коли будуть повністю перекладені](https://www.python.org/dev/peps/pep-0545/#add-translation-to-the-language-switcher):
-* `bugs`,
-* всі ресурси в каталозі `tutorial`,
-* `library/functions`.
-
-**Як переглянути останню збірку документації?**
-
-Завантажте останню створену документацію зі списку артефактів в останній дії GitHub (вкладка Actions).
-Переклади завантажуються з Transifex до цього репозиторію приблизно кожні півгодини.
-Документація на python.org оновлюється приблизно раз на день.
-
-**Канали зв'язку**
-
-* [Telegram-чат перекладачів](https://t.me/+dXwqHZ0KPKYyNDc6)
-* [Python translations working group](https://mail.python.org/mailman3/lists/translation.python.org/)
-* [Python Documentation Special Interest Group](https://www.python.org/community/sigs/current/doc-sig/)
-
-**Ліцензія**
-
-Запрошуючи вас до спільного створення проекту на платформі Transifex, ми пропонуємо договір на передачу ваших перекладів
-Python Software Foundation [по ліцензії CC0](https://creativecommons.org/publicdomain/zero/1.0/deed.uk).
-Натомість ви побачите, що ви є перекладачем тієї частини, яку ви переклали.
-Ви висловлюєте свою згоду з цією угодою, надаючи свою роботу для включення в документацію.
-
-**Оновлення локального перекладу**
-
-* `.github/scripts/manage_translation.py recreate_tx_config`
-* `.github/scripts/manage_translation.py fetch`
-* `.github/scripts/manage_translation.py recreate_readme`
-
-**Подяка**
-* Maciej Olko - Polish team
-* Julien Palard - French team
-* Tomo Cocoa - Japanese team
+from transifex.api import transifex_api
 
-**Внесок спільноти**  
 
-| Перекладач      | Кількість документів |  
-|:----------------|:--------------------:|  
-'''
-        )
+transifex_api.setup(auth=os.getenv('TX_TOKEN'))
 
-        file.writelines(
-            (f'|{t}|{c}|\n' for (t, c) in unique_translators.most_common())
-        )
+RESOURCE_NAME_MAP = {'glossary_': 'glossary'}
+
+ORGANISATION_ID = 'o:python-doc'
+PROJECT_ID = 'o:python-doc:p:python-newest'
+LANGUAGE_ID = 'l:uk'
+ORGANISATION = transifex_api.Organization.get(id=ORGANISATION_ID)
+PROJECT = transifex_api.Project.get(id=PROJECT_ID)
+LANGUAGE = transifex_api.Language.get(id=LANGUAGE_ID)
+
+
+def _slug_to_file_path(slug: str) -> Path:
+    """Set of rules how to transform slug to translation file path"""
+    file_path = RESOURCE_NAME_MAP.get(slug, slug)  # Legacy slug to file mapping
+    file_path = file_path.replace('--', '/')
+    if re.match(r'\d+_\d+', file_path):
+        file_path = file_path.replace('_', '.')
+    file_path = file_path + '.po'
+    return Path(file_path)
+
+
+def recreate_config() -> None:
+    """Regenerate Transifex client config for all resources."""
+    resources = transifex_api.Resource.filter(project=PROJECT).all()
+    with open('.tx/config', 'w') as fo:
+        fo.writelines(('[main]\n', 'host = https://api.transifex.com\n',))
+        for resource in resources:
+            path = _slug_to_file_path(resource.slug)
+            fo.writelines((
+                '\n',
+                f'[{resource.id}]\n',
+                f'file_filter = {path}\n',
+                'type = PO\n',
+                'source_lang = en\n',
+            ))
+
+
+def recreate_resource_stats() -> None:
+    """Create resource stats."""
+    stats = transifex_api.ResourceLanguageStats.filter(project=PROJECT, language=LANGUAGE).all()
+    with open('RESOURCE.md', 'w') as fo:
+        fo.writelines(('| Файл | Перекладено | Переглянуто | Вичитано |\n', '|:-----|:-----|:-----|:-----|\n'))
+        for stat in stats:
+            file_name = _slug_to_file_path(stat.id.split(':')[5])
+            translated_pct = round(100 * stat.attributes['translated_words'] / stat.attributes['total_words'], 1)
+            reviewed_pct = round(100 * stat.attributes['reviewed_words'] / stat.attributes['total_words'], 1)
+            proofread_pct = round(100 * stat.attributes['proofread_words'] / stat.attributes['total_words'], 1)
+            fo.writelines(f'| {file_name} | {translated_pct} % | {reviewed_pct} % | {proofread_pct} % |\n')
+
+
+def recreate_team_stats() -> None:
+    """Create contributor stats"""
+    members = transifex_api.TeamMembership.filter(organization=ORGANISATION, language=LANGUAGE).all()
+
+    users = {member.user.id: member.attributes['role'] for member in members}
+    translators = dict.fromkeys(users.keys(), 0)
+    reviewers = dict.fromkeys(users.keys(), 0)
+    proofreaders = dict.fromkeys(users.keys(), 0)
+
+    resources = transifex_api.Resource.filter(project=PROJECT).all()
+    for resource in resources:
+        translations = transifex_api.ResourceTranslation.filter(resource=resource, language=LANGUAGE).all()
+        for translation in translations:
+            if translation.relationships['translator']:
+                translators[translation.relationships['translator']['data']['id']] += 1
+            if translation.relationships['reviewer']:
+                reviewers[translation.relationships['reviewer']['data']['id']] += 1
+            if translation.relationships['proofreader']:
+                proofreaders[translation.relationships['proofreader']['data']['id']] += 1
+
+    with open('TEAM.md', 'w') as fo:
+        fo.writelines(('| | Роль | Переклав | Переглянув | Вичитав |\n', '|:---|:---|:---|:---|:---|\n',))
+        for user, role in users.items():
+            fo.writelines(f"| {user} | {role} | {translators[user]} | {reviewers[user]} | {proofreaders[user]} |\n")
+
+
+def fetch_translations():
+    """Fetch translations from Transifex, remove source lines."""
+    pull_return_code = os.system(f'tx pull -l uk --force --skip')
+    if pull_return_code != 0:
+        exit(pull_return_code)
 
 
 if __name__ == "__main__":
-    RUNNABLE_SCRIPTS = ('fetch', 'recreate_tx_config', 'recreate_readme')
+    RUNNABLE_SCRIPTS = ('recreate_config', 'recreate_resource_stats', 'recreate_team_stats',  'fetch_translations')
 
     parser = ArgumentParser()
     parser.add_argument('cmd', nargs=1, choices=RUNNABLE_SCRIPTS)

.github/workflows/update-and-build.yml

@@ -1,8 +1,9 @@
 on:
   schedule:
-    - cron: '0 0 * * *'
+    - cron: '0 6 * * *'
   push:
     branches: ['main']
+  workflow_dispatch:
 
 jobs:
   update-translation:
@@ -17,24 +18,19 @@ jobs:
       - uses: actions/setup-python@master
         with:
           python-version: 3
-      - run: sudo apt-get install -y gettext
       - run: curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
         working-directory: /usr/local/bin
-      - run: pip install requests
+      - run: pip install transifex-python
       - uses: actions/checkout@master
         with:
           ref: ${{ matrix.version }}
-      - run: .github/scripts/manage_translation.py recreate_tx_config
+      - run: .github/scripts/manage_translation.py recreate_config
         env:
           TX_TOKEN: ${{ secrets.TX_TOKEN }}
-      - run: .github/scripts/manage_translation.py fetch
+      - run: .github/scripts/manage_translation.py fetch_translations
         env:
           TX_TOKEN: ${{ secrets.TX_TOKEN }}
-      - run: .github/scripts/manage_translation.py recreate_readme
-        env:
-          TX_TOKEN: ${{ secrets.TX_TOKEN }}
-      - run: git config --local user.email 'dkazanzhy@gmail.com'
-        name: Run git config --local user.email '…'
+      - run: git config --local user.email github-actions@github.com
       - run: git config --local user.name "GitHub Action's update-translation job"
       - run: git add .
       - run: git commit -m 'Update translation from Transifex' || true
@@ -56,7 +52,7 @@ jobs:
     steps:
       - uses: actions/setup-python@master
         with:
-          python-version: '3.10'
+          python-version: 3
       - uses: actions/checkout@master
         with:
           repository: python/cpython

.tx/config

@@ -1,2 +1,2 @@
 [main]
-host = https://www.transifex.com
+host = https://api.transifex.com