diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 2daadaa..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -# CI on Windows via appveyor -environment: - - matrix: - ## MINGW - # - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - - PYTHON: "C:\\Miniconda35-x64" - PYTHON_VERSION: "3.5" - IS_CONDA: "yes" - -install: - - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - ## Print configuration for debugging. - # - - | - echo %PATH% - uname -a - where python pip pip2 pip3 pip34 - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - - - IF "%IS_CONDA%"=="yes" ( - conda info -a & - conda install --yes --quiet pip - ) - - pip install nose wheel coveralls - - ## For commits performed with the default user. - - | - git config --global user.email "travis@ci.com" - git config --global user.name "Travis Runner" - - - pip install -e . - -build: false - -test_script: - - IF "%PYTHON_VERSION%"=="3.5" ( - nosetests -v --with-coverage - ) ELSE ( - nosetests -v - ) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2fe73ca --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + +- package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..907698d --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,49 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + os: [ubuntu-latest] + experimental: [false] + include: + - python-version: "3.7" + os: ubuntu-22.04 + experimental: false + continue-on-error: ${{ matrix.experimental }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: ${{ matrix.experimental }} + - name: Install project and dependencies + run: | + python -m pip install --upgrade pip + pip install . + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pip install pytest + ulimit -n 48 + ulimit -n + pytest -v diff --git a/.gitignore b/.gitignore index e0b4e85..8b7da92 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ *.so .noseids *.sublime-workspace +*.egg-info diff --git a/.gitmodules b/.gitmodules index d85b15c..e73cded 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "smmap"] path = gitdb/ext/smmap - url = https://github.com/Byron/smmap.git + url = https://github.com/gitpython-developers/smmap.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1341a1d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -language: python -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) - -git: - # a higher depth is needed for one of the tests - lets fet - depth: 1000 -install: - - pip install coveralls -script: - - ulimit -n 48 - - ulimit -n - - nosetests -v --with-coverage -after_success: - - coveralls - diff --git a/AUTHORS b/AUTHORS index 490baad..6c7e9b9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,4 @@ Creator: Sebastian Thiel + +Contributors: + - Ram Rachum (@cool-RR) diff --git a/Makefile b/Makefile index 8cb323e..20436bb 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,12 @@ -PYTHON = python -SETUP = $(PYTHON) setup.py -TESTRUNNER = $(shell which nosetests) -TESTFLAGS = +.PHONY: all clean release force_release -all:: +all: @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all -release:: clean - # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag | sort -nr | head -n1)" - echo "HEAD SHA = $$(git rev-parse head)" - echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - make force_release - -force_release:: clean - git push --tags - python3 setup.py sdist bdist_wheel - twine upload -s -i byronimo@gmail.com dist/* - -doc:: - make -C doc/ html - -build:: - $(SETUP) build - $(SETUP) build_ext -i - -build_ext:: - $(SETUP) build_ext -i - -install:: - $(SETUP) install - -clean:: - $(SETUP) clean --all - rm -f *.so - -coverage:: build - PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=gitdb --with-coverage --cover-erase --cover-inclusive gitdb +clean: + rm -rf build/ dist/ .eggs/ .tox/ +force_release: clean + ./build-release.sh + twine upload dist/* + git push --tags origin master diff --git a/README.rst b/README.rst index 917b403..61ce28b 100644 --- a/README.rst +++ b/README.rst @@ -16,58 +16,54 @@ Installation :target: https://readthedocs.org/projects/gitdb/?badge=latest :alt: Documentation Status -From `PyPI `_ +From `PyPI `_:: pip install gitdb SPEEDUPS ======== -If you want to go up to 20% faster, you can install gitdb-speedups with: +If you want to go up to 20% faster, you can install gitdb-speedups with:: pip install gitdb-speedups +However, please note that gitdb-speedups is not currently maintained. + REQUIREMENTS ============ -* Python Nose - for running the tests +* smmap - declared as a dependency, automatically installed +* pytest - for running the tests SOURCE ====== -The source is available in a git repository at gitorious and github: + +The source is available in a git repository on GitHub: https://github.com/gitpython-developers/gitdb -Once the clone is complete, please be sure to initialize the submodules using +Once the clone is complete, please be sure to initialize the submodule using:: cd gitdb git submodule update --init -Run the tests with +Run the tests with:: - nosetests + pytest DEVELOPMENT =========== -.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master - :target: https://travis-ci.org/gitpython-developers/gitdb -.. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed - :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) -.. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png - :target: https://coveralls.io/r/gitpython-developers/gitdb -.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr - :target: http://www.issuestats.com/github/gitpython-developers/gitdb -.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue - :target: http://www.issuestats.com/github/gitpython-developers/gitdb +.. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg + :target: https://github.com/gitpython-developers/gitdb/actions -The library is considered mature, and not under active development. It's primary (known) use is in git-python. +The library is considered mature, and not under active development. Its primary (known) use is in GitPython. INFRASTRUCTURE ============== -* Mailing List - * http://groups.google.com/group/git-python +* Discussions + * https://github.com/gitpython-developers/GitPython/discussions * Issue Tracker * https://github.com/gitpython-developers/gitdb/issues diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..95389ff --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `gitdb` can be reported there. diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 0000000..5840e44 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This script builds a release. If run in a venv, it auto-installs its tools. +# You may want to run "make release" instead of running this script directly. + +set -eEu + +function release_with() { + $1 -m build --sdist --wheel +} + +if test -n "${VIRTUAL_ENV:-}"; then + deps=(build twine) # Install twine along with build, as we need it later. + echo "Virtual environment detected. Adding packages: ${deps[*]}" + pip install --quiet --upgrade "${deps[@]}" + echo 'Starting the build.' + release_with python +else + function suggest_venv() { + venv_cmd='python -m venv env && source env/bin/activate' + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" + } + trap suggest_venv ERR # This keeps the original exit (error) code. + echo 'Starting the build.' + release_with python3 # Outside a venv, use python3. +fi diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 22deb6d..b4340e4 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,104 @@ Changelog ######### +****** +4.0.12 +****** + +- various improvements - please see the release on GitHub for details. + +****** +4.0.11 +****** + +- various improvements - please see the release on GitHub for details. + +****** +4.0.10 +****** + +- improvements to the way external packages are imported. + +***** +4.0.9 +***** + +- re-release of 4.0.8 to get a valid signature. + +***** +4.0.8 +***** + +* drop support for python 3.4 and 3.5 due to EOL +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + +***** +4.0.7 +***** + +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + +***** +4.0.6 +***** + +* Bumped upper bound for smmap requirement + (`#67 `_, + `#68 `_) + +***** +4.0.5 +***** + +* Re-release of 4.0.4, with known signature + +***** +4.0.4 +***** + +* Support for PyOxidizer + +***** +4.0.2 +***** + +* Updated to release as Pure Python Wheel rather than Universal Wheel + (`#62 `_) + +***** +4.0.1 +***** + +* Switched back to the gitdb package name on PyPI and fixed the gitdb2 mirror package + (`#59 `_) +* Switched back to require smmap package and fixed version requirement to >= 3.0.1, < 4 + (`#59 `_) +* Updated smmap submodule + +*********** +3.0.3.post1 +*********** + +* Fixed changelogs for v3.0.2 and v3.0.3 + +***** +3.0.3 +***** + +* Changed ``force_bytes`` to use UTF-8 encoding by default + (`#49 `_) +* Restricted smmap2 version requirement to < 3 +* Updated requirements.txt + +***** +3.0.2 +***** + +* Removed Python 2 compatibility shims + (`#56 `_) + ***** 0.6.1 ***** diff --git a/doc/source/conf.py b/doc/source/conf.py index 68d9a3f..b387f60 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # GitDB documentation build configuration file, created by # sphinx-quickstart on Wed Jun 30 00:01:32 2010. @@ -38,8 +37,8 @@ master_doc = 'index' # General information about the project. -project = u'GitDB' -copyright = u'2011, Sebastian Thiel' +project = 'GitDB' +copyright = '2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -120,7 +119,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['.static'] +#html_static_path = ['.static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -172,8 +171,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'GitDB.tex', u'GitDB Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'GitDB.tex', 'GitDB Documentation', + 'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project deleted file mode 100644 index d0e2e51..0000000 --- a/etc/sublime-text/gitdb.sublime-project +++ /dev/null @@ -1,44 +0,0 @@ -{ - "folders": - [ - // GITDB - //////// - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "gitdb/ext", - "dist", - "doc/build", - ".tox" - ] - }, - // SMMAP - //////// - { - "follow_symlinks": true, - "path": "../../gitdb/ext/smmap", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "dist", - "doc/build", - ".tox" - ] - }, - ] -} diff --git a/gitdb.pro.user b/gitdb.pro.user index 398cb70..3ca1e21 100644 --- a/gitdb.pro.user +++ b/gitdb.pro.user @@ -233,8 +233,7 @@ - /usr/bin/nosetests - -s + /usr/bin/pytest gitdb/test/test_pack.py 2 diff --git a/gitdb/__init__.py b/gitdb/__init__.py index e184e4b..1fb7df8 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -1,38 +1,15 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Initialize the object database module""" -import sys -import os - -#{ Initialization - - -def _init_externals(): - """Initialize external projects by putting them into the path""" - for module in ('smmap',): - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - - try: - __import__(module) - except ImportError: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) - # END verify import - # END handel imports - -#} END initialization - -_init_externals() - __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (4, 0, 12) __version__ = '.'.join(str(i) for i in version_info) - # default imports from gitdb.base import * from gitdb.db import * diff --git a/gitdb/base.py b/gitdb/base.py index 42e71d0..9a23a4f 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with basic data structures - they are designed to be lightweight and fast""" from gitdb.util import bin_to_hex diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index 0a2a46a..20fd228 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import * from gitdb.db.loose import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2d7b9fa..7312fe0 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains implementations of database retrieveing objects""" from gitdb.util import ( join, @@ -22,7 +22,7 @@ __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') -class ObjectDBR(object): +class ObjectDBR: """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" @@ -33,6 +33,9 @@ def __contains__(self, sha): #{ Query Interface def has_object(self, sha): """ + Whether the object identified by the given 20 bytes + binary sha is contained in the database + :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") @@ -60,7 +63,7 @@ def sha_iter(self): #} END query interface -class ObjectDBW(object): +class ObjectDBW: """Defines an interface to create objects in the database""" @@ -82,6 +85,8 @@ def set_ostream(self, stream): def ostream(self): """ + Return the output stream + :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream @@ -100,7 +105,7 @@ def store(self, istream): #} END edit interface -class FileDBBase(object): +class FileDBBase: """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" @@ -112,7 +117,7 @@ def __init__(self, root_path): **Note:** The base will not perform any accessablity checking as the base might not yet be accessible, but become accessible before the first access.""" - super(FileDBBase, self).__init__() + super().__init__() self._root_path = root_path #{ Interface @@ -128,7 +133,7 @@ def db_path(self, rela_path): #} END interface -class CachingDB(object): +class CachingDB: """A database which uses caches to speed-up access""" @@ -171,7 +176,7 @@ def _set_cache_(self, attr): elif attr == '_db_cache': self._db_cache = dict() else: - super(CompoundDB, self)._set_cache_(attr) + super()._set_cache_(attr) def _db_query(self, sha): """:return: database containing the given 20 byte sha diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 7a43d72..a1ed142 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import ( CompoundDB, ObjectDBW, @@ -39,7 +39,7 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): def __init__(self, root_path): """Initialize ourselves on a git objects directory""" - super(GitDB, self).__init__(root_path) + super().__init__(root_path) def _set_cache_(self, attr): if attr == '_dbs' or attr == '_loose_db': @@ -68,7 +68,7 @@ def _set_cache_(self, attr): # finally set the value self._loose_db = loose_db else: - super(GitDB, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs #{ ObjectDBW interface diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 192c524..e6765cd 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -1,14 +1,15 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ +from contextlib import suppress + from gitdb.db.base import ( FileDBBase, ObjectDBR, ObjectDBW ) - from gitdb.exc import ( BadObject, AmbiguousObjectName @@ -33,10 +34,8 @@ bin_to_hex, exists, chmod, - isdir, isfile, remove, - mkdir, rename, dirname, basename, @@ -50,11 +49,12 @@ stream_copy ) -from gitdb.utils.compat import MAXSIZE from gitdb.utils.encoding import force_bytes import tempfile import os +import sys +import time __all__ = ('LooseObjectDB', ) @@ -75,7 +75,7 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): new_objects_mode = int("644", 8) def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) + super().__init__(root_path) self._hexsha_to_file = dict() # Additional Flags - might be set to 0 after the first failure # Depending on the root, this might work for some mounts, for others not, which @@ -93,10 +93,8 @@ def readable_db_object_path(self, hexsha): """ :return: readable object path to the object identified by hexsha :raise BadObject: If the object file does not exist""" - try: + with suppress(KeyError): return self._hexsha_to_file[hexsha] - except KeyError: - pass # END ignore cache misses # try filesystem @@ -138,12 +136,12 @@ def _map_loose_object(self, sha): # try again without noatime try: return file_contents_ro_filepath(db_path) - except OSError: - raise BadObject(sha) + except OSError as new_e: + raise BadObject(sha) from new_e # didn't work because of our flag, don't try it again self._fd_open_flags = 0 else: - raise BadObject(sha) + raise BadObject(sha) from e # END handle error # END exception handling @@ -151,7 +149,7 @@ def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" if stream is not None and not isinstance(stream, Sha1Writer): raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) + return super().set_ostream(stream) def info(self, sha): m = self._map_loose_object(sha) @@ -196,7 +194,7 @@ def store(self, istream): if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version - stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size) + stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, @@ -208,7 +206,7 @@ def store(self, istream): # END assure target stream is closed except: if tmp_path: - os.remove(tmp_path) + remove(tmp_path) raise # END assure tmpfile removal on error @@ -222,23 +220,31 @@ def store(self, istream): if tmp_path: obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) + os.makedirs(obj_dir, exist_ok=True) # END handle destination directory - # rename onto existing doesn't work on windows - if os.name == 'nt': - if isfile(obj_path): - remove(tmp_path) - else: - rename(tmp_path, obj_path) - # end rename only if needed + # rename onto existing doesn't work on NTFS + if isfile(obj_path): + remove(tmp_path) else: rename(tmp_path, obj_path) - # END handle win32 + # end rename only if needed + + # Ensure rename is actually done and file is stable + # Retry up to 14 times - quadratic wait & retry in ms. + # The total maximum wait time is 1000ms, which should be vastly enough for the + # OS to return and commit the file to disk. + for backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: + with suppress(PermissionError): + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + break + time.sleep(backoff_ms / 1000.0) + else: + raise PermissionError( + "Impossible to apply `chmod` to file {}".format(obj_path) + ) - # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rwrr - chmod(obj_path, self.new_objects_mode) # END handle dry_run istream.binsha = hex_to_bin(hexsha) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 8711334..d4772fd 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains the MemoryDatabase implementation""" from gitdb.db.loose import LooseObjectDB from gitdb.db.base import ( @@ -37,7 +37,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): exists in the target storage before introducing actual IO""" def __init__(self): - super(MemoryDB, self).__init__() + super().__init__() self._db = LooseObjectDB("path/doesnt/matter") # maps 20 byte shas to their OStream objects @@ -74,24 +74,22 @@ def stream(self, sha): # rewind stream for the next one to read ostream.stream.seek(0) return ostream - except KeyError: - raise BadObject(sha) + except KeyError as e: + raise BadObject(sha) from e # END exception handling def size(self): return len(self._cache) def sha_iter(self): - try: - return self._cache.iterkeys() - except AttributeError: - return self._cache.keys() + return self._cache.keys() #{ Interface def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly **Note:** the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount of input shas, one or more objects did already exist in odb""" count = 0 diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 1e37d73..274ea59 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing a database to deal with packs""" from gitdb.db.base import ( FileDBBase, @@ -18,7 +18,6 @@ ) from gitdb.pack import PackEntity -from gitdb.utils.compat import xrange from functools import reduce @@ -40,7 +39,7 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): _sort_interval = 500 def __init__(self, root_path): - super(PackedDB, self).__init__(root_path) + super().__init__(root_path) # list of lists with three items: # * hits - number of times the pack was hit with a request # * entity - Pack entity instance @@ -107,7 +106,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in xrange(index.size()): + for index in range(index.size()): yield sha_by_index(index) # END for each index # END for each entity @@ -132,7 +131,7 @@ def store(self, istream): def update_cache(self, force=False): """ - Update our cache with the acutally existing packs on disk. Add new ones, + Update our cache with the actually existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones :param force: If True, the cache will be updated even though the directory diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 2bb1de7..bd30156 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import codecs from gitdb.db.base import ( CompoundDB, @@ -20,7 +20,7 @@ class ReferenceDB(CompoundDB): ObjectDBCls = None def __init__(self, ref_file): - super(ReferenceDB, self).__init__() + super().__init__() self._ref_file = ref_file def _set_cache_(self, attr): @@ -28,7 +28,7 @@ def _set_cache_(self, attr): self._dbs = list() self._update_dbs_from_ref_file() else: - super(ReferenceDB, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs def _update_dbs_from_ref_file(self): @@ -44,7 +44,7 @@ def _update_dbs_from_ref_file(self): try: with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] - except (OSError, IOError): + except OSError: pass # END handle alternates @@ -79,4 +79,4 @@ def _update_dbs_from_ref_file(self): def update_cache(self, force=False): # re-read alternates and update databases self._update_dbs_from_ref_file() - return super(ReferenceDB, self).update_cache(force) + return super().update_cache(force) diff --git a/gitdb/exc.py b/gitdb/exc.py index 947e5d8..752dafd 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -1,10 +1,21 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with common exceptions""" from gitdb.util import to_hex_sha +__all__ = [ + 'AmbiguousObjectName', + 'BadName', + 'BadObject', + 'BadObjectType', + 'InvalidDBRoot', + 'ODBError', + 'ParseError', + 'UnsupportedOperation', + 'to_hex_sha', +] class ODBError(Exception): """All errors thrown by the object database""" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 91d506e..8f82e6c 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 91d506e120d4a0f98cbef202325e301c632445c5 +Subproject commit 8f82e6c19661f9b735cc55cc89031a189e408894 diff --git a/gitdb/fun.py b/gitdb/fun.py index 8ca38c8..a272e5c 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains basic c-functions which usually contain performance critical code Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" @@ -16,7 +16,6 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import izip, buffer, xrange, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -101,10 +100,10 @@ def delta_chunk_apply(dc, bbuf, write): :param write: write method to call with data to write""" if dc.data is None: # COPY DATA FROM SOURCE - write(buffer(bbuf, dc.so, dc.ts)) + write(bbuf[dc.so:dc.so + dc.ts]) else: # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? + # what's faster: if + 4 function calls or just a write with a slice ? # Considering data can be larger than 127 bytes now, it should be worth it if dc.ts < len(dc.data): write(dc.data[:dc.ts]) @@ -114,7 +113,7 @@ def delta_chunk_apply(dc, bbuf, write): # END handle chunk mode -class DeltaChunk(object): +class DeltaChunk: """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -264,7 +263,7 @@ def compress(self): # if first_data_index is not None: nd = StringIO() # new data so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i - 1): + for x in range(first_data_index, i - 1): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data @@ -293,7 +292,7 @@ def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size :param target_size: if not -1, the total size of the chain must be target_size - :raise AssertionError: if the size doen't match""" + :raise AssertionError: if the size doesn't match""" if target_size > -1: assert self[-1].rbound() == target_size assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size @@ -314,7 +313,7 @@ def check_integrity(self, target_size=-1): right.next() # this is very pythonic - we might have just use index based access here, # but this could actually be faster - for lft, rgt in izip(left, right): + for lft, rgt in zip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair @@ -332,7 +331,7 @@ def connect_with_next_base(self, bdcl): cannot be changed by any of the upcoming bases anymore. Once all our chunks are marked like that, we can stop all processing :param bdcl: data chunk list being one of our bases. They must be fed in - consequtively and in order, towards the earliest ancestor delta + consecutively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams if False is returned""" nfc = 0 # number of frozen chunks @@ -424,20 +423,12 @@ def pack_object_header_info(data): type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size - if PY3: - while c & 0x80: - c = byte_ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop - else: - while c & 0x80: - c = ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop # end performance at expense of maintenance ... return (type_id, size, i) @@ -450,28 +441,16 @@ def create_pack_object_header(obj_type, obj_size): :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte - if PY3: - hdr = bytearray() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr.append(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - # END until size is consumed - hdr.append(c) - else: - hdr = bytes() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - # END until size is consumed - hdr += chr(c) + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + # END until size is consumed + hdr.append(c) # end handle interpreter return hdr @@ -484,26 +463,15 @@ def msb_size(data, offset=0): i = 0 l = len(data) hit_msb = False - if PY3: - while i < l: - c = data[i + offset] - size |= (c & 0x7f) << i * 7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range - else: - while i < l: - c = ord(data[i + offset]) - size |= (c & 0x7f) << i * 7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range + while i < l: + c = data[i + offset] + size |= (c & 0x7f) << i * 7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") @@ -656,100 +624,55 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf_size: size for the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf - if PY3: - while i < delta_buf_size: - c = db[i] - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = db[i] - i += 1 - if (c & 0x02): - cp_off |= (db[i] << 8) - i += 1 - if (c & 0x04): - cp_off |= (db[i] << 16) - i += 1 - if (c & 0x08): - cp_off |= (db[i] << 24) - i += 1 - if (c & 0x10): - cp_size = db[i] - i += 1 - if (c & 0x20): - cp_size |= (db[i] << 8) - i += 1 - if (c & 0x40): - cp_size |= (db[i] << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i + c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - else: - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i + c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - # end save byte_ord call and prevent performance regression in py2 + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(src_buf[cp_off:cp_off + cp_size]) + elif c: + write(db[i:i + c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" diff --git a/gitdb/pack.py b/gitdb/pack.py index 115d943..e559e11 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains PackIndexFile and PackFile implementations""" import zlib @@ -62,12 +62,6 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import ( - izip, - buffer, - xrange, - to_bytes -) import tempfile import array @@ -119,7 +113,7 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + stream = DecompressMemMapReader(data[total_rela_offset:], False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: @@ -176,7 +170,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): #} END utilities -class IndexWriter(object): +class IndexWriter: """Utility to cache index information, allowing to write all information later in one go to the given stream @@ -207,7 +201,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[byte_ord(t[0][0])] += 1 # END prepare fanout - for i in xrange(255): + for i in range(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i + 1] += v @@ -230,7 +224,7 @@ def write(self, pack_sha, write): if ofs > 0x7fffffff: tmplist.append(ofs) ofs = 0x80000000 + len(tmplist) - 1 - # END hande 64 bit offsets + # END handle 64 bit offsets sha_write(pack('>L', ofs & 0xffffffff)) # END for each offset @@ -263,13 +257,13 @@ class PackIndexFile(LazyMixin): index_version_default = 2 def __init__(self, indexpath): - super(PackIndexFile, self).__init__() + super().__init__() self._indexpath = indexpath def close(self): mman.force_map_handle_removal_win(self._indexpath) self._cursor = None - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -376,7 +370,7 @@ def _read_fanout(self, byte_offset): d = self._cursor.map() out = list() append = out.append - for i in xrange(256): + for i in range(256): append(unpack_from('>L', d, byte_offset + i * 4)[0]) # END for each entry return out @@ -410,14 +404,14 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.frombytes(self._cursor.map()[self._pack_offset:self._pack_64_offset]) # networkbyteorder to something array likes more if sys.byteorder == 'little': a.byteswap() return a else: - return tuple(self.offset(index) for index in xrange(self.size())) + return tuple(self.offset(index) for index in range(self.size())) # END handle version def sha_to_index(self, sha): @@ -512,7 +506,7 @@ class PackFile(LazyMixin): """A pack is a file written according to the Version 2 for git packs As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + packs therefore is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. **Note:** at some point, this might be implemented using streams as well, or @@ -534,7 +528,7 @@ def __init__(self, packpath): def close(self): mman.force_map_handle_removal_win(self._packpath) self._cursor = None - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() @@ -696,7 +690,7 @@ def _set_cache_(self, attr): iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) next(iter_offsets_plus_one) - consecutive = izip(iter_offsets, iter_offsets_plus_one) + consecutive = zip(iter_offsets, iter_offsets_plus_one) offset_map = dict(consecutive) @@ -716,7 +710,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in xrange(self._index.size()): + for index in range(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index @@ -838,7 +832,7 @@ def is_valid_stream(self, sha, use_crc=False): while cur_pos < next_offset: rbound = min(cur_pos + chunk_size, next_offset) size = rbound - cur_pos - this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + this_crc_value = crc_update(pack_data[cur_pos:cur_pos + size], this_crc_value) cur_pos += size # END window size loop @@ -882,7 +876,11 @@ def collect_streams_at_offset(self, offset): stream = streams[-1] while stream.type_id in delta_types: if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(to_bytes(stream.delta_info)) + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + if isinstance(stream.delta_info, memoryview): + sindex = self._index.sha_to_index(stream.delta_info.tobytes()) + else: + sindex = self._index.sha_to_index(stream.delta_info) if sindex is None: break stream = self._pack.stream(self._index.offset(sindex)) diff --git a/gitdb/stream.py b/gitdb/stream.py index 2f4c12d..1e0be84 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO @@ -27,11 +27,9 @@ ) from gitdb.const import NULL_BYTE, BYTE_SPACE -from gitdb.utils.compat import buffer from gitdb.utils.encoding import force_bytes has_perf_mod = False -PY26 = sys.version_info[:2] < (2, 7) try: from gitdb_speedups._perf import apply_delta as c_apply_delta has_perf_mod = True @@ -142,7 +140,7 @@ def data(self): def close(self): """Close our underlying stream of compressed bytes if this was allowed during initialization :return: True if we closed the underlying stream - :note: can be called safely + :note: can be called safely """ if self._close: if hasattr(self._m, 'close'): @@ -221,13 +219,13 @@ def read(self, size=-1): # END clamp size if size == 0: - return bytes() + return b'' # END handle depletion # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream - dat = bytes() + dat = b'' if self._buf: if self._buflen >= size: # have enough data @@ -278,7 +276,7 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) + indata = self._m[self._cws:self._cwe] # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) @@ -289,14 +287,14 @@ def read(self, size=-1): # if we hit the end of the stream # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. # They are thorough, and I assume it is truly working. - # Why is this logic as convoluted as it is ? Please look at the table in + # Why is this logic as convoluted as it is ? Please look at the table in # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. - # Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch - # will be the one that works. - # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if PY26 or ((zlib.ZLIB_VERSION == '1.2.7' or zlib.ZLIB_VERSION == '1.2.5') and not sys.platform == 'darwin'): + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) @@ -414,7 +412,7 @@ def _set_cache_brute_(self, attr): buf = dstream.read(512) # read the header information + X offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) + buffer_info_list.append((buf[offset:], offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream @@ -555,7 +553,7 @@ def size(self): #{ W Streams -class Sha1Writer(object): +class Sha1Writer: """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" @@ -652,7 +650,7 @@ class FDCompressedSha1Writer(Sha1Writer): exc = IOError("Failed to write all bytes to filedescriptor") def __init__(self, fd): - super(FDCompressedSha1Writer, self).__init__() + super().__init__() self.fd = fd self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) @@ -679,7 +677,7 @@ def close(self): #} END stream interface -class FDStream(object): +class FDStream: """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream @@ -713,7 +711,7 @@ def close(self): close(self._fd) -class NullStream(object): +class NullStream: """A stream that does nothing but providing a stream interface. Use it like /dev/null""" diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 8a681e4..03bd406 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/test/db/__init__.py b/gitdb/test/db/__init__.py index 8a681e4..03bd406 100644 --- a/gitdb/test/db/__init__.py +++ b/gitdb/test/db/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 528bcc1..408dd8c 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, @@ -23,7 +23,6 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type -from gitdb.utils.compat import xrange from io import BytesIO @@ -45,7 +44,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in xrange(ni): + for i in range(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) @@ -103,12 +102,12 @@ def _assert_object_writing(self, db): assert ostream.type == str_blob_type assert ostream.size == len(data) else: - self.failUnlessRaises(BadObject, db.info, sha) - self.failUnlessRaises(BadObject, db.stream, sha) + self.assertRaises(BadObject, db.info, sha) + self.assertRaises(BadObject, db.stream, sha) # DIRECT STREAM COPY # our data hase been written in object format to the StringIO - # we pasesd as output stream. No physical database representation + # we passed as output stream. No physical database representation # was created. # Test direct stream copy of object streams, the result must be # identical to what we fed in diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index acc0f15..73ac1a0 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import os from gitdb.test.db.lib import ( TestDBBase, @@ -43,7 +43,7 @@ def test_reading(self): assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha # END for each sha - self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + self.assertRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 9c25a02..295e2ee 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory @@ -32,5 +32,5 @@ def test_basics(self, path): assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha # END for each sha - self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + self.assertRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') # raises if no object could be found diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index eb563c0..882e54f 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 9694238..bd07906 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, @@ -16,7 +16,7 @@ import random import sys -from nose.plugins.skip import SkipTest +import pytest class TestPackDB(TestDBBase): @@ -24,7 +24,7 @@ class TestPackDB(TestDBBase): @with_packs_rw def test_writing(self, path): if sys.platform == "win32": - raise SkipTest("FIXME: Currently fail on windows") + pytest.skip("FIXME: Currently fail on windows") pdb = PackedDB(path) @@ -80,8 +80,8 @@ def test_writing(self, path): # END for each sha to find # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... + # but in our pack, there is no ambiguous ... # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) + self.assertRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 2049698..0816e64 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, @@ -23,7 +23,7 @@ def make_alt_file(self, alt_path, alt_list): The list can be empty""" with open(alt_path, "wb") as alt_file: for alt in alt_list: - alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) + alt_file.write(alt.encode("utf-8") + b"\n") @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ab1842d..8e60234 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -1,10 +1,9 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Utilities used in ODB testing""" from gitdb import OStream -from gitdb.utils.compat import xrange import sys import random @@ -41,7 +40,7 @@ class TestBase(unittest.TestCase): @classmethod def setUpClass(cls): try: - super(TestBase, cls).setUpClass() + super().setUpClass() except AttributeError: pass @@ -59,21 +58,6 @@ def setUpClass(cls): #{ Decorators -def skip_on_travis_ci(func): - """All tests decorated with this one will raise SkipTest when run on travis ci. - Use it to workaround difficult to solve issues - NOTE: copied from bcore (https://github.com/Byron/bcore)""" - @wraps(func) - def wrapper(self, *args, **kwargs): - if 'TRAVIS' in os.environ: - import nose - raise nose.SkipTest("Cannot run on travis-ci") - # end check for travis ci - return func(self, *args, **kwargs) - # end wrapper - return wrapper - - def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test succeeds, but leave it otherwise to aid additional debugging""" @@ -86,7 +70,7 @@ def wrapper(self): try: return func(self, path) except Exception: - sys.stderr.write("Test {}.{} failed, output is at {!r}\n".format(type(self).__name__, func.__name__, path)) + sys.stderr.write(f"Test {type(self).__name__}.{func.__name__} failed, output is at {path!r}\n") keep = True raise finally: @@ -151,13 +135,13 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes // 4 - producer = xrange(actual_size) + producer = range(actual_size) if randomize: producer = list(producer) random.shuffle(producer) # END randomize a = array('i', producer) - return a.tostring() + return a.tobytes() def make_object(type, data): @@ -177,7 +161,7 @@ def make_memory_file(size_in_bytes, randomize=False): #{ Stream Utilities -class DummyStream(object): +class DummyStream: def __init__(self): self.was_read = False diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index fa4dd20..36916ed 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains library functions""" from gitdb.test.lib import TestBase diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index fc8d9d5..fc3d334 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -1,9 +1,8 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Performance tests for object store""" -from __future__ import print_function from gitdb.test.performance.lib import ( TestBigRepoR @@ -17,8 +16,6 @@ from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB -from gitdb.utils.compat import xrange -from gitdb.test.lib import skip_on_travis_ci import sys import os @@ -27,7 +24,6 @@ class TestPackedDBPerformance(TestBigRepoR): - @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -80,7 +76,6 @@ def test_pack_random_access(self): print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) - @skip_on_travis_ci def test_loose_correctness(self): """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back into the loose object db (memory). @@ -107,7 +102,6 @@ def test_loose_correctness(self): mdb._cache.clear() # end for each sha to copy - @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time @@ -118,7 +112,7 @@ def test_correctness(self): for entity in pdb.entities(): pack_verify = entity.is_valid_stream sha_by_index = entity.index().sha - for index in xrange(entity.index().size()): + for index in range(entity.index().size()): try: assert pack_verify(sha_by_index(index), use_crc=crc) count += 1 diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 76f0f4a..80c798b 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -1,9 +1,8 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Specific test for pack streams only""" -from __future__ import print_function from gitdb.test.performance.lib import ( TestBigRepoR @@ -12,7 +11,6 @@ from gitdb.db.pack import PackedDB from gitdb.stream import NullStream from gitdb.pack import PackEntity -from gitdb.test.lib import skip_on_travis_ci import os import sys @@ -34,7 +32,6 @@ def write(self, d): class TestPackStreamingPerformance(TestBigRepoR): - @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well @@ -61,7 +58,6 @@ def test_pack_writing(self): print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) - @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 92d28e4..fb10871 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -1,9 +1,8 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Performance data streaming performance""" -from __future__ import print_function from gitdb.test.performance.lib import TestBigRepoR from gitdb.db import LooseObjectDB @@ -20,7 +19,6 @@ from gitdb.test.lib import ( make_memory_file, with_rw_directory, - skip_on_travis_ci ) @@ -44,7 +42,6 @@ class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB - @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 519cdfd..17906c9 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" from gitdb.test.lib import ( TestBase, @@ -73,7 +73,7 @@ def test_streams(self): # test deltapackstream dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream + assert dpostream.stream is stream dpostream.read(5) stream._assert() assert stream.bytes == 5 @@ -92,7 +92,7 @@ def test_streams(self): assert istream.size == s istream.size = s * 2 - istream.size == s * 2 + assert istream.size == s * 2 assert istream.type == str_blob_type istream.type = "something" assert istream.type == "something" diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 6e80bf5..3b4c908 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with examples from the tutorial section of the docs""" import os from gitdb.test.lib import TestBase @@ -32,7 +32,7 @@ def test_base(self): pass # END ignore exception if there are no loose objects - data = "my data".encode("ascii") + data = b"my data" istream = IStream("blob", len(data), BytesIO(data)) # the object does not yet have a sha diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 24e2a31..e723482 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test everything about packs reading and writing""" from gitdb.test.lib import ( TestBase, @@ -25,14 +25,8 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from gitdb.utils.compat import xrange -try: - from itertools import izip -except ImportError: - izip = zip - -from nose import SkipTest +import pytest import os import tempfile @@ -63,7 +57,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in xrange(index.size()): + for oidx in range(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) @@ -79,7 +73,7 @@ def _assert_index_file(self, index, version, size): assert index.partial_sha_to_index(sha[:l], l * 2) == oidx # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) + self.assertRaises(ValueError, index.partial_sha_to_index, "\0", 2) def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 @@ -107,7 +101,7 @@ def _assert_pack_file(self, pack, version, size): dstream = DeltaApplyReader.new(streams) except ValueError: # ignore these, old git versions use only ref deltas, - # which we havent resolved ( as we are without an index ) + # which we haven't resolved ( as we are without an index ) # Also ignore non-delta streams continue # END get deltastream @@ -155,7 +149,7 @@ def test_pack_entity(self, rw_dir): pack_objs.extend(entity.stream_iter()) count = 0 - for info, stream in izip(entity.info_iter(), entity.stream_iter()): + for info, stream in zip(entity.info_iter(), entity.stream_iter()): count += 1 assert info.binsha == stream.binsha assert len(info.binsha) == 20 @@ -248,8 +242,8 @@ def rewind_streams(): # END for each info assert count == len(pack_objs) entity.close() - + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack - raise SkipTest() + pytest.skip('not implemented') diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 9626825..1e7e941 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" from gitdb.test.lib import ( @@ -115,13 +115,13 @@ def test_decompress_reader(self): def test_sha_writer(self): writer = Sha1Writer() - assert 2 == writer.write("hi".encode("ascii")) + assert 2 == writer.write(b"hi") assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 # make sure it does something ;) prev_sha = writer.sha() - writer.write("hi again".encode("ascii")) + writer.write(b"hi again") assert writer.sha() != prev_sha def test_compressed_writer(self): @@ -135,7 +135,7 @@ def test_compressed_writer(self): ostream.close() # its closed already - self.failUnlessRaises(OSError, os.close, fd) + self.assertRaises(OSError, os.close, fd) # read everything back, compare to data we zip fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 847bdab..166b33c 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" import tempfile import os @@ -40,8 +40,8 @@ def test_lockedfd(self): lockfilepath = lfd._lockfilepath() # cannot end before it was started - self.failUnlessRaises(AssertionError, lfd.rollback) - self.failUnlessRaises(AssertionError, lfd.commit) + self.assertRaises(AssertionError, lfd.rollback) + self.assertRaises(AssertionError, lfd.commit) # open for writing assert not os.path.isfile(lockfilepath) @@ -77,7 +77,7 @@ def test_lockedfd(self): wfdstream = lfd.open(write=True, stream=True) # this time as stream assert os.path.isfile(lockfilepath) # another one fails - self.failUnlessRaises(IOError, olfd.open) + self.assertRaises(IOError, olfd.open) wfdstream.write(new_data.encode("ascii")) lfd.commit() diff --git a/gitdb/typ.py b/gitdb/typ.py index 98d15f3..314db50 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing information about types known to the database""" str_blob_type = b'blob' diff --git a/gitdb/util.py b/gitdb/util.py index d680f97..bb6d879 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import binascii import os import mmap @@ -94,7 +94,7 @@ def remove(*args, **kwargs): #{ compatibility stuff ... -class _RandomAccessBytesIO(object): +class _RandomAccessBytesIO: """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" @@ -131,7 +131,7 @@ def byte_ord(b): #{ Routines -def make_sha(source=''.encode("ascii")): +def make_sha(source=b''): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ @@ -151,7 +151,7 @@ def allocate_memory(size): try: return mmap.mmap(-1, size) # read-write by default - except EnvironmentError: + except OSError: # setup real memory instead # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on @@ -174,7 +174,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # supports stream and random access try: return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - except EnvironmentError: + except OSError: # python 2.4 issue, 0 wants to be the actual size return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) # END handle python 2.4 @@ -182,7 +182,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): pass # END exception handling - # read manully + # read manually contents = os.read(fd, os.fstat(fd).st_size) if stream: return _RandomAccessBytesIO(contents) @@ -234,7 +234,7 @@ def to_bin_sha(sha): #{ Utilities -class LazyMixin(object): +class LazyMixin: """ Base class providing an interface to lazily retrieve attribute values upon @@ -248,7 +248,7 @@ class LazyMixin(object): def __getattr__(self, attr): """ Whenever an attribute is requested that we do not know, we allow it - to be created and set. Next time the same attribute is reqeusted, it is simply + to be created and set. Next time the same attribute is requested, it is simply returned from our dict/slots. """ self._set_cache_(attr) # will raise in case the cache was not created @@ -266,7 +266,7 @@ def _set_cache_(self, attr): pass -class LockedFD(object): +class LockedFD: """ This class facilitates a safe read and write operation to a file on disk. @@ -326,13 +326,13 @@ def open(self, write=False, stream=False): else: self._fd = fd # END handle file descriptor - except OSError: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + except OSError as e: + raise OSError("Lock at %r could not be obtained" % self._lockfilepath()) from e # END handle lock retrieval # open actual file if required if self._fd is None: - # we could specify exlusive here, as we obtained the lock anyway + # we could specify exclusive here, as we obtained the lock anyway try: self._fd = os.open(self._filepath, os.O_RDONLY | binary) except: diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py deleted file mode 100644 index a7899cb..0000000 --- a/gitdb/utils/compat.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys - -PY3 = sys.version_info[0] == 3 - -try: - from itertools import izip - xrange = xrange -except ImportError: - # py3 - izip = zip - xrange = range -# end handle python version - -try: - # Python 2 - buffer = buffer - memoryview = buffer - # Assume no memory view ... - def to_bytes(i): - return i -except NameError: - # Python 3 has no `buffer`; only `memoryview` - # However, it's faster to just slice the object directly, maybe it keeps a view internally - def buffer(obj, offset, size=None): - if size is None: - # return memoryview(obj)[offset:] - return obj[offset:] - else: - # return memoryview(obj)[offset:offset+size] - return obj[offset:offset + size] - # end buffer reimplementation - # smmap can return memory view objects, which can't be compared as buffers/bytes can ... - def to_bytes(i): - if isinstance(i, memoryview): - return i.tobytes() - return i - - memoryview = memoryview - -try: - MAXSIZE = sys.maxint -except AttributeError: - MAXSIZE = sys.maxsize diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 4d270af..b534ef7 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -1,31 +1,18 @@ -from gitdb.utils import compat - -if compat.PY3: - string_types = (str, ) - text_type = str -else: - string_types = (basestring, ) - text_type = unicode - - -def force_bytes(data, encoding="ascii"): +def force_bytes(data, encoding="utf-8"): if isinstance(data, bytes): return data - if isinstance(data, string_types): + if isinstance(data, str): return data.encode(encoding) return data def force_text(data, encoding="utf-8"): - if isinstance(data, text_type): + if isinstance(data, str): return data if isinstance(data, bytes): return data.decode(encoding) - if compat.PY3: - return text_type(data, encoding) - else: - return text_type(data) + return str(data, encoding) diff --git a/requirements.txt b/requirements.txt index ed4898e..1b2e11d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=0.8.3 +smmap>=3.0.1,<6 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3c6e79c..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 diff --git a/setup.py b/setup.py index 27bb754..3a91543 100755 --- a/setup.py +++ b/setup.py @@ -7,11 +7,11 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (4, 0, 12) __version__ = '.'.join(str(i) for i in version_info) setup( - name="gitdb2", + name="gitdb", version=__version__, description="Git Object Database", author=__author__, @@ -20,9 +20,9 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap2 >= 2.0.0'], + install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=3.7', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -34,11 +34,14 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", ] )