Switch task queue to procrastinate based on psql

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.24.0
+current_version = 5.0.0-rc5
 tag_name = {new_version}
 commit = True
 tag = True

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.github/workflows/build.yml

@@ -82,3 +82,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Run e2e test
+        working-directory: ./e2e
+        run: bash -c ./test_e2e.sh

.github/workflows/docker-base.yml

@@ -2,8 +2,8 @@ name: Build ingest-file-base
 
 on:
   workflow_dispatch: {}
-  schedule:
-    - cron: "0 0 * * *"
+  # schedule:
+  #   - cron: "0 0 * * *"
   push:
     paths:
       - Dockerfile.base
@@ -28,7 +28,8 @@ jobs:
             type=ref,event=branch
             type=semver,pattern={{version}}
             type=sha
-            type=raw,value=latest
+            type=raw,value=cache
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags') }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
@@ -44,7 +45,7 @@ jobs:
         with:
           context: .
           file: ./Dockerfile.base
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64 #,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -1,5 +1,10 @@
+contrib/*.bin
+data/archive
 data/model_type_prediction.ftz
+debug.sqlite3
 data/servicelayer-archive
+# documentation
+site
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.pre-commit-config.yaml

@@ -5,20 +5,21 @@
 # * Run "pre-commit install".
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
       - id: check-merge-conflict
       - id: check-symlinks
       - id: check-toml
       - id: check-yaml
+        exclude: "mkdocs.yml"
       - id: debug-statements
       - id: end-of-file-fixer
       - id: mixed-line-ending
         args: ["--fix=lf"]
       - id: trailing-whitespace
-        exclude: ".bumpversion.cfg"  # wtf
+        exclude: ".bumpversion.cfg" # wtf
 
   # - repo: https://github.com/asottile/pyupgrade
   #   rev: v3.10.1
@@ -78,7 +79,7 @@ repos:
     rev: 1.9.0
     hooks:
       - id: poetry-export
-        args: ["--without-hashes", "-o", "requirements.txt"]
+        args: ["--without-hashes", "--with", "main", "-o", "requirements.txt"]
       - id: poetry-export
         args:
           ["--without-hashes", "--only", "dev", "-o", "requirements-dev.txt"]

Dockerfile

@@ -1,15 +1,22 @@
 FROM ghcr.io/openaleph/ingest-file-base:latest
 
+# uncomment when running on Apple Silicon
+# ENV LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libgomp.so.1
+
 COPY . /ingestors
+RUN rm -rf /ingestors/tests
 WORKDIR /ingestors
-RUN pip3 install --no-cache-dir -r /ingestors/requirements.txt
-RUN pip3 install --no-cache-dir /ingestors
+
+RUN pip3 install --no-cache-dir --no-deps -r /ingestors/requirements.txt
+RUN pip3 install --no-deps --no-cache-dir /ingestors
 
 ENV ARCHIVE_TYPE=file \
     ARCHIVE_PATH=/data \
-    FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \
+    OPENALEPH_DB_URI=postgresql://aleph:aleph@postgres/aleph \
     REDIS_URL=redis://redis:6379/0 \
     TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata
 
-USER app
-CMD ingestors process
+ENV PROCRASTINATE_APP="ingestors.tasks.app"
+
+CMD ["procrastinate", "worker", "-q", "ingest"]

Dockerfile.base

@@ -12,12 +12,12 @@ RUN apt-get -qq -y update \
     # python deps (mostly to install their dependencies)
     python3-pip python3-dev python3-pil \
     # tesseract
-    tesseract-ocr libtesseract-dev libleptonica-dev pkg-config\
+    tesseract-ocr libtesseract-dev libleptonica-dev pkg-config \
     # libraries
     libxslt1-dev libpq-dev libldap2-dev libsasl2-dev \
     zlib1g-dev libicu-dev libxml2-dev \
     # package tools
-    unrar p7zip-full \
+    unrar \
     # audio & video metadata
     libmediainfo-dev \
     # image processing, djvu
@@ -116,41 +116,22 @@ ENV LANG='en_US.UTF-8' \
     OMP_THREAD_LIMIT='1' \
     OPENBLAS_NUM_THREADS='1'
 
-ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libgomp.so.1"
+# force compile tesserocr 2.6.2 with C++ 14
+# to make it compatible with Tesseract 5
+RUN pip download --no-binary=:all: "tesserocr==2.6.2" \
+    && tar -xzf tesserocr-2.6.2.tar.gz \
+    && sed -i "s/-std=c++11/-std=c++14/" tesserocr-2.6.2/setup.py \
+    && cd tesserocr-2.6.2 \
+    && CXXFLAGS="-std=c++14" pip install --no-cache-dir .
 
 # tesseract 5
 ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata
 
 RUN groupadd -g 1000 -r app \
     && useradd -m -u 1000 -s /bin/false -g app app
 
-# Download the ftm-typepredict model
-RUN mkdir /models/ && \
-    curl -o "/models/model_type_prediction.ftz" "https://public.data.occrp.org/develop/models/types/type-08012020-7a69d1b.ftz"
-
 RUN pip3 install --no-cache-dir --prefer-binary --upgrade pip
 RUN pip3 install --no-cache-dir --prefer-binary --upgrade setuptools wheel
 
-# Install spaCy
-RUN pip3 install --no-cache-dir spacy
 # Install PyICU
 RUN pip3 install --no-binary=:pyicu: pyicu
-# Install TesserOCR
-RUN pip3 install --no-binary=:tesserocr: tesserocr
-
-# Install default (small) spaCy models
-RUN python3 -m spacy download en_core_web_sm
-RUN python3 -m spacy download de_core_news_sm
-RUN python3 -m spacy download fr_core_news_sm
-RUN python3 -m spacy download es_core_news_sm
-RUN python3 -m spacy download ru_core_news_sm
-RUN python3 -m spacy download pt_core_news_sm
-RUN python3 -m spacy download ro_core_news_sm
-RUN python3 -m spacy download mk_core_news_sm
-RUN python3 -m spacy download el_core_news_sm
-RUN python3 -m spacy download pl_core_news_sm
-RUN python3 -m spacy download it_core_news_sm
-RUN python3 -m spacy download lt_core_news_sm
-RUN python3 -m spacy download nl_core_news_sm
-RUN python3 -m spacy download nb_core_news_sm
-RUN python3 -m spacy download da_core_news_sm

Dockerfile.test

@@ -1,20 +1,26 @@
 FROM ghcr.io/openaleph/ingest-file-base:latest
 
+# uncomment when running on Apple Silicon
 # ENV LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libgomp.so.1
 
 COPY . /ingestors
 WORKDIR /ingestors
-RUN pip3 install --no-cache-dir -r /ingestors/requirements.txt
-RUN pip3 install --no-cache-dir /ingestors
 
-RUN pip3 install -r /ingestors/requirements-dev.txt
+RUN pip3 install --no-cache-dir --no-deps -r /ingestors/requirements.txt
+RUN pip3 install --no-deps --no-cache-dir /ingestors
+
+RUN pip3 install --no-deps -r /ingestors/requirements-dev.txt
+RUN pip3 install --no-cache-dir procrastinate==3.2.2
 RUN chown -R app:app /ingestors
 
 ENV ARCHIVE_TYPE=file \
     ARCHIVE_PATH=/data \
     FTM_STORE_URI=postgresql://aleph:aleph@postgres/aleph \
     REDIS_URL=redis://redis:6379/0 \
-    TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata
+    TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata \
+    DEBUG=1
+
+ENV PROCRASTINATE_APP="ingestors.tasks.app"
 
-USER app
-CMD ingestors process
+CMD ["pytest"]

Makefile

@@ -1,5 +1,7 @@
 INGEST=ghcr.io/openaleph/ingest-file
+INGEST=ghcr.io/openaleph/ingest-file
 COMPOSE=docker compose
+COMPOSE_E2E=docker compose -f docker-compose.e2e.yml
 DOCKER=$(COMPOSE) run --rm ingest-file
 
 .PHONY: build
@@ -9,12 +11,18 @@ all: build shell
 build:
 	$(COMPOSE) build --no-rm --parallel
 
-build-macos:
-	DOCKER_BUILDKIT=0 COMPOSE_DOCKER_CLI_BUILD=0 $(COMPOSE) build --no-rm --parallel
-
 build-base:
 	docker build . -f Dockerfile.base -t ghcr.io/openaleph/ingest-file-base:latest
 
+build-cache:
+	docker build . --cache-from ghcr.io/openaleph/ingest-file:cache -t ghcr.io/openaleph/ingest-file:cache
+
+build-test:
+	$(COMPOSE) build test-ingest-file
+
+build-macos:
+	DOCKER_BUILDKIT=0 COMPOSE_DOCKER_CLI_BUILD=0 $(COMPOSE) build --no-rm --parallel
+
 services:
 	$(COMPOSE) up -d --remove-orphans postgres redis
 
@@ -30,8 +38,11 @@ format:
 format-check:
 	black --check .
 
-test: build services
-	PYTHONDEVMODE=1 PYTHONTRACEMALLOC=1 $(DOCKER) pytest --cov=ingestors --cov-report html --cov-report term
+test: build-test services
+	PYTHONDEVMODE=1 PYTHONTRACEMALLOC=1 $(COMPOSE) run --rm test-ingest-file pytest
+
+test-e2e: build services
+	$(COMPOSE_E2E) run --rm ingest-file
 
 restart: build
 	$(COMPOSE) up --force-recreate --no-deps --detach ingest-file
@@ -53,3 +64,7 @@ clean:
 dev:
 	python3 -m pip install --upgrade pip
 	python3 -m pip install -q -r requirements-dev.txt
+
+documentation:
+	mkdocs build
+	aws --profile nbg1 --endpoint-url https://s3.investigativedata.org s3 sync ./site s3://openaleph.org/docs/lib/ingest-file

README.md

@@ -1,26 +1,17 @@
-# ingestors
+[![Docs](https://img.shields.io/badge/docs-live-brightgreen)](https://openaleph.org/docs/lib/ingest-file/)
+[![Python test and package](https://github.com/openaleph/ingest-file/actions/workflows/build.yml/badge.svg)](https://github.com/openaleph/ingest-file/actions/workflows/build.yml)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Coverage Status](https://coveralls.io/repos/github/openaleph/ingest-file/badge.svg?branch=main)](https://coveralls.io/github/openaleph/ingest-file?branch=main)
+[![AGPLv3+ License](https://img.shields.io/pypi/l/ftmq)](./LICENSE)
+[![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://pydantic.dev)
 
-``ingestors`` extract useful information from documents of different types in
-a structured standard format. It retains folder structures across directories,
-compressed archives and emails. The extracted data is formatted as Follow the
-Money (FtM) entities, ready for import into Aleph, or processing as an object
-graph.
+# ingest-file
 
-Supported file types:
+``ingest-file`` extract useful information from documents of different types in a structured standard format. It retains folder structures across directories, compressed archives and emails. The extracted data is formatted as [Follow the Money (FtM)](https://followthemoney.tech) entities, ready for import into [OpenAleph](https://openaleph.org), or processing as an object graph.
 
-* Plain text
-* Images
-* Web pages, XML documents
-* PDF files
-* Emails (Outlook, plain text)
-* Archive files (ZIP, Rar, etc.)
+## Documentation
 
-Other features:
-
-* Extendable and composable using classes and mixins.
-* Generates FollowTheMoney objects to a database as result objects.
-* Lightweight worker-style support for logging, failures and callbacks.
-* Thoroughly tested.
+https://openaleph.org/docs/lib/ingest-file
 
 ## Development environment
 

VERSION

@@ -1 +1 @@
-3.24.0
+5.0.0-rc5

contrib/install_deb.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# works for current debian testing (trixie/sid)
+
+apt-get install build-essential locales ca-certificates \
+    python3-pip python3-dev python3-pil \
+    tesseract-ocr libtesseract-dev libleptonica-dev pkg-config\
+    libxslt1-dev libpq-dev libldap2-dev libsasl2-dev \
+    zlib1g-dev libicu-dev libxml2-dev \
+    unrar p7zip-full \
+    libmediainfo-dev \
+    imagemagick-6-common imagemagick mdbtools djvulibre-bin \
+    libtiff5-dev libjpeg-dev libfreetype-dev libwebp-dev \
+    libtiff-tools ghostscript librsvg2-bin jbig2dec \
+    pst-utils \
+    tesseract-ocr-eng \
+    tesseract-ocr-swa \
+    tesseract-ocr-swe \
+    tesseract-ocr-fil \
+    tesseract-ocr-tur \
+    tesseract-ocr-ukr \
+    tesseract-ocr-nld \
+    tesseract-ocr-nor \
+    tesseract-ocr-pol \
+    tesseract-ocr-por \
+    tesseract-ocr-ron \
+    tesseract-ocr-rus \
+    tesseract-ocr-slk \
+    tesseract-ocr-slv \
+    tesseract-ocr-spa \
+    tesseract-ocr-sqi \
+    tesseract-ocr-srp \
+    tesseract-ocr-ind \
+    tesseract-ocr-isl \
+    tesseract-ocr-ita \
+    tesseract-ocr-kan \
+    tesseract-ocr-kat \
+    tesseract-ocr-khm  \
+    tesseract-ocr-lav \
+    tesseract-ocr-lit \
+    tesseract-ocr-mkd \
+    tesseract-ocr-mya \
+    tesseract-ocr-mlt \
+    tesseract-ocr-msa \
+    tesseract-ocr-est \
+    tesseract-ocr-fin \
+    tesseract-ocr-fra \
+    tesseract-ocr-frk \
+    tesseract-ocr-heb \
+    tesseract-ocr-hin \
+    tesseract-ocr-hrv \
+    tesseract-ocr-hye \
+    tesseract-ocr-hun \
+    tesseract-ocr-bul \
+    tesseract-ocr-cat \
+    tesseract-ocr-ces \
+    tesseract-ocr-nep \
+    tesseract-ocr-dan \
+    tesseract-ocr-deu \
+    tesseract-ocr-ell \
+    tesseract-ocr-afr \
+    tesseract-ocr-ara \
+    tesseract-ocr-aze \
+    tesseract-ocr-bel \
+    tesseract-ocr-uzb \
+    libreoffice fonts-opensymbol hyphen-fr hyphen-de \
+    hyphen-en-us hyphen-it hyphen-ru fonts-dejavu fonts-dejavu-core fonts-dejavu-extra \
+    fonts-droid-fallback fonts-dustin fonts-f500 fonts-fanwood fonts-freefont-ttf \
+    fonts-liberation fonts-lmodern fonts-lyx fonts-sil-gentium fonts-texgyre \
+    fonts-tlwg-purisa