gitdb now uses 20 byte shas internally only, reducing the need to convert shas around all the time, saving previous function calls, and memory after all

Author: Byron

base.py

@@ -1,7 +1,6 @@
 """Module with basic data structures - they are designed to be lightweight and fast"""
 from util import (
-		to_hex_sha,
-		to_bin_sha,
+		bin_to_hex,
 		zlib
 	)
 
@@ -17,13 +16,13 @@
 #{ ODB Bases
 
 class OInfo(tuple):
-	"""Carries information about an object in an ODB, provdiing information 
-	about the sha of the object, the type_string as well as the uncompressed size
+	"""Carries information about an object in an ODB, provding information 
+	about the binary sha of the object, the type_string as well as the uncompressed size
 	in bytes.
 	
 	It can be accessed using tuple notation and using attribute access notation::
 	
-		assert dbi[0] == dbi.sha
+		assert dbi[0] == dbi.binsha
 		assert dbi[1] == dbi.type
 		assert dbi[2] == dbi.size
 	
@@ -38,18 +37,14 @@ def __init__(self, *args):
 
 	#{ Interface 
 	@property
-	def sha(self):
+	def binsha(self):
+		""":return: our sha as binary, 20 bytes"""
 		return self[0]
-		
+	
 	@property
 	def hexsha(self):
 		""":return: our sha, hex encoded, 40 bytes"""
-		return to_hex_sha(self[0])
-		
-	@property
-	def binsha(self):
-		""":return: our sha as binary, 20 bytes"""
-		return to_bin_sha(self[0])
+		return bin_to_hex(self[0])
 
 	@property
 	def type(self):
@@ -197,16 +192,10 @@ def __init__(self, type, size, stream, sha=None):
 		list.__init__(self, (sha, type, size, stream, None))
 
 	#{ Interface 
-	
 	@property
 	def hexsha(self):
 		""":return: our sha, hex encoded, 40 bytes"""
-		return to_hex_sha(self[0])
-		
-	@property
-	def binsha(self):
-		""":return: our sha as binary, 20 bytes"""
-		return to_bin_sha(self[0])
+		return bin_to_hex(self[0])
 
 	def _error(self):
 		""":return: the error that occurred when processing the stream, or None"""
@@ -231,13 +220,13 @@ def read(self, size=-1):
 
 	#{  interface
 
-	def _set_sha(self, sha):
-		self[0] = sha
+	def _set_binsha(self, binsha):
+		self[0] = binsha
 
-	def _sha(self):
+	def _binsha(self):
 		return self[0]
 
-	sha = property(_sha, _set_sha)
+	binsha = property(_binsha, _set_binsha)
 
 
 	def _type(self):
@@ -280,9 +269,13 @@ def __init__(self, sha, exc):
 		tuple.__init__(self, (sha, exc))
 
 	@property
-	def sha(self):
+	def binsha(self):
 		return self[0]
 
+	@property
+	def hexsha(self):
+		return bin_to_hex(self[0])
+		
 	@property
 	def error(self):
 		""":return: exception instance explaining the failure"""

db/base.py

@@ -2,8 +2,7 @@
 from gitdb.util import (
 		pool,
 		join, 
-		LazyMixin, 
-		to_bin_sha
+		LazyMixin 
 	)
 
 from gitdb.exc import BadObject
@@ -20,31 +19,30 @@
 
 class ObjectDBR(object):
 	"""Defines an interface for object database lookup.
-	Objects are identified either by hex-sha (40 bytes) or 
-	by sha (20 bytes)"""
+	Objects are identified either by their 20 byte bin sha"""
 
 	def __contains__(self, sha):
 		return self.has_obj
 
 	#{ Query Interface 
 	def has_object(self, sha):
 		"""
-		:return: True if the object identified by the given 40 byte hexsha or 20 bytes
+		:return: True if the object identified by the given 20 bytes
 			binary sha is contained in the database"""
 		raise NotImplementedError("To be implemented in subclass")
 
 	def has_object_async(self, reader):
 		"""Return a reader yielding information about the membership of objects
 		as identified by shas
-		:param reader: Reader yielding 20 byte or 40 byte shas.
+		:param reader: Reader yielding 20 byte shas.
 		:return: async.Reader yielding tuples of (sha, bool) pairs which indicate
 			whether the given sha exists in the database or not"""
 		task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha)))
 		return pool.add_task(task) 
 
 	def info(self, sha):
 		""" :return: OInfo instance
-		:param sha: 40 bytes hexsha or 20 bytes binary sha
+		:param sha: bytes binary sha
 		:raise BadObject:"""
 		raise NotImplementedError("To be implemented in subclass")
 
@@ -57,7 +55,7 @@ def info_async(self, reader):
 
 	def stream(self, sha):
 		""":return: OStream instance
-		:param sha: 40 bytes hexsha or 20 bytes binary sha
+		:param sha: 20 bytes binary sha
 		:raise BadObject:"""
 		raise NotImplementedError("To be implemented in subclass")
 
@@ -192,11 +190,10 @@ def _set_cache_(self, attr):
 			super(CompoundDB, self)._set_cache_(attr)
 
 	def _db_query(self, sha):
-		""":return: database containing the given 20 or 40 byte sha
+		""":return: database containing the given 20 byte sha
 		:raise BadObject:"""
 		# most databases use binary representations, prevent converting 
 		# it everytime a database is being queried
-		sha = to_bin_sha(sha)
 		try:
 			return self._db_cache[sha]
 		except KeyError:

db/loose.py

@@ -25,8 +25,8 @@
 from gitdb.util import (
 		file_contents_ro_filepath,
 		ENOENT,
-		to_hex_sha,
 		hex_to_bin,
+		bin_to_hex,
 		exists,
 		chmod,
 		isdir,
@@ -100,7 +100,7 @@ def _map_loose_object(self, sha):
 		"""
 		:return: memory map of that file to allow random read access
 		:raise BadObject: if object could not be located"""
-		db_path = self.db_path(self.object_path(to_hex_sha(sha)))
+		db_path = self.db_path(self.object_path(bin_to_hex(sha)))
 		try:
 			return file_contents_ro_filepath(db_path, flags=self._fd_open_flags)
 		except OSError,e:
@@ -109,11 +109,11 @@ def _map_loose_object(self, sha):
 				try:
 					return file_contents_ro_filepath(db_path)
 				except OSError:
-					raise BadObject(to_hex_sha(sha))
+					raise BadObject(sha)
 				# didn't work because of our flag, don't try it again
 				self._fd_open_flags = 0
 			else:
-				raise BadObject(to_hex_sha(sha))
+				raise BadObject(sha)
 			# END handle error
 		# END exception handling
 		try:
@@ -144,7 +144,7 @@ def stream(self, sha):
 
 	def has_object(self, sha):
 		try:
-			self.readable_db_object_path(to_hex_sha(sha))
+			self.readable_db_object_path(bin_to_hex(sha))
 			return True
 		except BadObject:
 			return False
@@ -158,7 +158,7 @@ def store(self, istream):
 			# open a tmp file to write the data to
 			fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path)
 
-			if istream.sha is None:
+			if istream.binsha is None:
 				writer = FDCompressedSha1Writer(fd)
 			else:
 				writer = FDStream(fd)
@@ -167,7 +167,7 @@ def store(self, istream):
 
 		try:
 			try:
-				if istream.sha is not None:
+				if istream.binsha is not None:
 					# copy as much as possible, the actual uncompressed item size might
 					# be smaller than the compressed version
 					stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size)
@@ -187,7 +187,7 @@ def store(self, istream):
 		# END assure tmpfile removal on error
 
 		hexsha = None
-		if istream.sha:
+		if istream.binsha:
 			hexsha = istream.hexsha
 		else:
 			hexsha = writer.sha(as_hex=True)
@@ -206,7 +206,7 @@ def store(self, istream):
 			chmod(obj_path, 0444)
 		# END handle dry_run
 
-		istream.sha = hexsha
+		istream.binsha = hex_to_bin(hexsha)
 		return istream
 
 	def sha_iter(self):

db/mem.py

@@ -10,7 +10,6 @@
 							IStream,
 						)
 
-from gitdb.util import to_bin_sha
 from gitdb.exc import (
 						BadObject,
 						UnsupportedOperation
@@ -54,22 +53,21 @@ def store(self, istream):
 		# don't provide a size, the stream is written in object format, hence the 
 		# header needs decompression
 		decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) 
-		self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream)
+		self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream)
 
 		return istream
 
 	def store_async(self, reader):
 		raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access")
 
 	def has_object(self, sha):
-		return to_bin_sha(sha) in self._cache
+		return sha in self._cache
 
 	def info(self, sha):
 		# we always return streams, which are infos as well
 		return self.stream(sha)
 
 	def stream(self, sha):
-		sha = to_bin_sha(sha)
 		try:
 			ostream = self._cache[sha]
 			# rewind stream for the next one to read

db/pack.py

@@ -5,10 +5,7 @@
 						CachingDB
 				)
 
-from gitdb.util import (
-							to_bin_sha, 
-							LazyMixin
-						)
+from gitdb.util import LazyMixin
 
 from gitdb.exc import (
 							BadObject,
@@ -65,7 +62,6 @@ def _pack_info(self, sha):
 			self._sort_entities()
 		# END update sorting
 
-		sha = to_bin_sha(sha)
 		for item in self._entities:
 			index = item[2](sha)
 			if index is not None:

test/db/lib.py

@@ -41,15 +41,15 @@ def _assert_object_writing_simple(self, db):
 			istream = IStream(str_blob_type, len(data), StringIO(data))
 			new_istream = db.store(istream)
 			assert new_istream is istream
-			assert db.has_object(istream.sha)
+			assert db.has_object(istream.binsha)
 
-			info = db.info(istream.sha)
+			info = db.info(istream.binsha)
 			assert isinstance(info, OInfo)
 			assert info.type == istream.type and info.size == istream.size
 
-			stream = db.stream(istream.sha)
+			stream = db.stream(istream.binsha)
 			assert isinstance(stream, OStream)
-			assert stream.sha == info.sha and stream.type == info.type
+			assert stream.binsha == info.binsha and stream.type == info.type
 			assert stream.read() == data
 		# END for each item
 
@@ -80,10 +80,10 @@ def _assert_object_writing(self, db):
 
 				# store returns same istream instance, with new sha set
 				my_istream = db.store(istream)
-				sha = istream.sha
+				sha = istream.binsha
 				assert my_istream is istream
 				assert db.has_object(sha) != dry_run
-				assert len(sha) == 40		# for now we require 40 byte shas as default
+				assert len(sha) == 20	
 
 				# verify data - the slow way, we want to run code
 				if not dry_run:
@@ -107,12 +107,12 @@ def _assert_object_writing(self, db):
 					# identical to what we fed in
 					ostream.seek(0)
 					istream.stream = ostream
-					assert istream.sha is not None
-					prev_sha = istream.sha
+					assert istream.binsha is not None
+					prev_sha = istream.binsha
 
 					db.set_ostream(ZippedStoreShaWriter())
 					db.store(istream)
-					assert istream.sha == prev_sha
+					assert istream.binsha == prev_sha
 					new_ostream = db.ostream()
 
 					# note: only works as long our store write uses the same compression
@@ -143,12 +143,12 @@ def istream_generator(offset=0, ni=ni):
 
 		for stream in istreams:
 			assert stream.error is None
-			assert len(stream.sha) == 40
+			assert len(stream.binsha) == 20
 			assert isinstance(stream, IStream)
 		# END assert each stream
 
 		# test has-object-async - we must have all previously added ones
-		reader = IteratorReader( istream.sha for istream in istreams )
+		reader = IteratorReader( istream.binsha for istream in istreams )
 		hasobject_reader = db.has_object_async(reader)
 		count = 0
 		for sha, has_object in hasobject_reader:
@@ -158,7 +158,7 @@ def istream_generator(offset=0, ni=ni):
 		assert count == ni
 
 		# read the objects we have just written
-		reader = IteratorReader( istream.sha for istream in istreams )
+		reader = IteratorReader( istream.binsha for istream in istreams )
 		ostream_reader = db.stream_async(reader)
 
 		# read items individually to prevent hitting possible sys-limits
@@ -171,7 +171,7 @@ def istream_generator(offset=0, ni=ni):
 		assert count == ni
 
 		# get info about our items
-		reader = IteratorReader( istream.sha for istream in istreams )
+		reader = IteratorReader( istream.binsha for istream in istreams )
 		info_reader = db.info_async(reader)
 
 		count = 0
@@ -186,7 +186,7 @@ def istream_generator(offset=0, ni=ni):
 		# add 2500 items, and obtain their output streams
 		nni = 2500
 		reader = IteratorReader(istream_generator(offset=ni, ni=nni))
-		istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ]
+		istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ]
 
 		istream_reader = db.store_async(reader)
 		istream_reader.set_post_cb(istream_to_sha)