From 4a667686e6f983379ce7abcc3234ae44dd886dec Mon Sep 17 00:00:00 2001 From: PythonCoderAS <13932583+PythonCoderAS@users.noreply.github.com> Date: Tue, 28 Nov 2023 23:33:05 -0500 Subject: [PATCH 1/6] Fully fix build --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 582bba6..f5f0609 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,8 @@ COPY --from=generate-requirements /app/requirements.txt ./ RUN ["pip", "install", "-r", "requirements.txt"] -COPY src . +COPY src/main.py . -ENTRYPOINT [ "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers" ] +ENTRYPOINT [ "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers" ] EXPOSE 8000 \ No newline at end of file From 7f57d1f7d0f64ef528bae6496a51ca083c6c3678 Mon Sep 17 00:00:00 2001 From: PythonCoderAS <13932583+PythonCoderAS@users.noreply.github.com> Date: Tue, 28 Nov 2023 23:34:15 -0500 Subject: [PATCH 2/6] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0fdfac4..98372e5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -34,7 +34,6 @@ jobs: - linux/arm/v7 - linux/arm64/v8 - linux/386 - - windows/amd64 permissions: contents: read packages: write From effb84bdee13b99dbacbcef9204897f382c76634 Mon Sep 17 00:00:00 2001 From: PythonCoderAS <13932583+PythonCoderAS@users.noreply.github.com> Date: Tue, 28 Nov 2023 23:45:10 -0500 Subject: [PATCH 3/6] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 98372e5..c60c9d8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,8 +30,6 @@ jobs: matrix: platform: - linux/amd64 - - linux/arm/v5 - - linux/arm/v7 - linux/arm64/v8 - linux/386 permissions: From ce9b4dc3f40499c0166b56c7cf902087e8a7a2e0 Mon Sep 17 00:00:00 2001 From: PythonCoderAS <13932583+PythonCoderAS@users.noreply.github.com> Date: Tue, 28 Nov 2023 23:55:53 -0500 Subject: [PATCH 4/6] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c60c9d8..164ebd7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -43,14 +43,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - # Install the cosign tool except on PR - # https://github.com/sigstore/cosign-installer - - name: Install cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 - with: - cosign-release: 'v2.1.1' - # Set up BuildKit Docker container builder to be able to build # multi-platform images and export cache # https://github.com/docker/setup-buildx-action @@ -88,18 +80,3 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: ${{ matrix.platform }} - - # Sign the resulting Docker image digest except on PRs. - # This will only write to the public Rekor transparency log when the Docker - # repository is public to avoid leaking data. If you would like to publish - # transparency data even for private images, pass --force to cosign below. - # https://github.com/sigstore/cosign - - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} - env: - # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable - TAGS: ${{ steps.meta.outputs.tags }} - DIGEST: ${{ steps.build-and-push.outputs.digest }} - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} From 2fe5173b7fd247c927d24779538e8adddfdd4ebb Mon Sep 17 00:00:00 2001 From: PythonCoderAS <13932583+PythonCoderAS@users.noreply.github.com> Date: Tue, 28 Nov 2023 23:56:49 -0500 Subject: [PATCH 5/6] Session fixups --- src/main.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main.py b/src/main.py index be8da4b..3ccf470 100644 --- a/src/main.py +++ b/src/main.py @@ -140,15 +140,13 @@ async def repeat_url_worker(): created_at: datetime.datetime = None while True: curtime = datetime.datetime.now(tz=datetime.timezone.utc) - async with async_session() as session: + async with async_session() as session, session.begin(): stmt = select(RepeatURL).where(RepeatURL.active_since <= curtime).order_by(RepeatURL.created_at) - async with session.begin(): - result = await session.scalars(stmt) - jobs = result.all() + result = await session.scalars(stmt) + jobs = result.all() stmt2 = select(URL.url).join(Job).where(URL.url.in_([job.url.url for job in jobs]) & (Job.completed == None) & (Job.failed == None)) - async with session.begin(): - result = await session.scalars(stmt2) - existing_jobs = result.all() + result = await session.scalars(stmt2) + existing_jobs = result.all() queued: list[Job] = [] for job in jobs: if (not job.url.last_seen or job.url.last_seen + datetime.timedelta(seconds=job.interval) < curtime) and job.url.url not in existing_jobs: # Job can be re-queued @@ -158,8 +156,7 @@ async def repeat_url_worker(): created_at = curtime queued.append(Job(url=job.url, priority=10, batches=[batch, job.batch])) if queued: - async with session.begin(): - session.add_all(queued) + session.add_all(queued) await asyncio.sleep(60) From 7a5b566da11493b3bb96004c14510bb812248c63 Mon Sep 17 00:00:00 2001 From: PythonCoderAS <13932583+PythonCoderAS@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:45:38 -0500 Subject: [PATCH 6/6] Fix bugs --- .github/workflows/docker-publish.yml | 9 +-------- src/main.py | 6 ++++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 164ebd7..a5f73ac 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -25,13 +25,6 @@ env: jobs: build: runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - platform: - - linux/amd64 - - linux/arm64/v8 - - linux/386 permissions: contents: read packages: write @@ -79,4 +72,4 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - platforms: ${{ matrix.platform }} + platforms: linux/amd64,linux/arm64/v8,linux/386 diff --git a/src/main.py b/src/main.py index 3ccf470..5a7045a 100644 --- a/src/main.py +++ b/src/main.py @@ -125,8 +125,9 @@ async def url_worker(): await session.execute(update(URL).where(URL.id == next_job.url.id).values(last_seen=saved_dt)) await session.execute(update(Job).where(Job.id == next_job.id).values(completed=saved_dt)) break - except Exception as e: - raise e + except Exception: + pass + await asyncio.sleep(10) else: # Ran out of retries, try again async with session.begin(): if next_job.retry < 4: @@ -154,6 +155,7 @@ async def repeat_url_worker(): # batch = await Batch.objects.create() batch = Batch() created_at = curtime + session.add(batch) queued.append(Job(url=job.url, priority=10, batches=[batch, job.batch])) if queued: session.add_all(queued)