Cached revisions

setup.py

@@ -14,7 +14,7 @@
         "long_description (%s)\n" % readme_file)
     sys.exit(1)
 
-install_requires = ['Pygments', 'mock']
+install_requires = ['Pygments', 'mock', 'lockfile>=0.9.1']
 if sys.version_info < (2, 7):
     install_requires.append('unittest2')
 tests_require = install_requires + ['dulwich', 'mercurial']

vcs/backends/base.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import with_statement
+import os
+import gzip
 import datetime
+import lockfile
 import itertools
 
 from vcs.utils import author_name, author_email
@@ -45,10 +49,12 @@ class BaseRepository(object):
             tags as list of changesets
     """
     scm = None
+    use_revisions_cache = False
     DEFAULT_BRANCH_NAME = None
     EMPTY_CHANGESET = '0' * 40
 
-    def __init__(self, repo_path, create=False, **kwargs):
+    def __init__(self, repo_path, create=False, src_url=None,
+            use_revisions_cache=False):
         """
         Initializes repository. Raises RepositoryError if repository could
         not be find at the given ``repo_path`` or directory at ``repo_path``
@@ -60,6 +66,8 @@ def __init__(self, repo_path, create=False, **kwargs):
           would be cloned; requires ``create`` parameter to be set to True -
           raises RepositoryError if src_url is set and create evaluates to
           False
+        :param use_revisions_cache: if set to True, would try to use cached
+          revisions list (and saves it if cache does not exist).
         """
         raise NotImplementedError
 
@@ -79,6 +87,60 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def _get_all_revisions(self):
+        raise NotImplementedError
+
+    def invalidate_revisions(self):
+        """
+        Marks ``revisions`` attribute to be re-fetched next time it's accessed.
+        """
+        self._revisions = None
+
+    def _revisions_get(self):
+        """
+        Returns list of revisions' ids, in ascending order.  Being lazy
+        attribute allows external tools to inject shas from cache.
+        """
+        if getattr(self, '_revisions', None) is None:
+            if self.use_revisions_cache:
+                cache_path = self.get_revisions_cache_path()
+                if not os.path.isfile(cache_path):
+                    self.cache_revisions()
+                self._revisions = self.get_cached_revisions()
+            else:
+                self._revisions = self._get_all_revisions()
+        return self._revisions
+
+    def _revisions_set(self, revs):
+        self._revisions = revs
+
+    revisions = property(_revisions_get, _revisions_set)
+
+    def get_revisions_cache_path(self):
+        cache_filename = '.vcs.%s.revisions.cache' % self.scm
+        return os.path.join(self.path, cache_filename)
+
+    def cache_revisions(self):
+        with self.get_revisions_lock():
+            revisions = self._get_all_revisions()
+            try:
+                fout = gzip.open(self.get_revisions_cache_path(), 'w')
+                for revision in revisions:
+                    fout.write('%s\n' % revision)
+            finally:
+                fout.close()
+
+    def get_cached_revisions(self):
+        return gzip.open(self.get_revisions_cache_path()).read().splitlines()
+
+    def get_revisions_lock(self):
+        """
+        Returns ``lockfile.LockFile`` lock.
+        """
+        lock_filename = '.vcs.%s.revisions.lock' % self.scm
+        lockpath = os.path.join(self.path, lock_filename)
+        return lockfile.LockFile(lockpath)
+
     @LazyProperty
     def alias(self):
         for k, v in settings.BACKENDS.items():

vcs/backends/git/inmemory.py

@@ -150,10 +150,10 @@ def commit(self, message, author, parents=None, branch=None, date=None,
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
-        # Update vcs repository object & recreate dulwich repo
-        self.repository.revisions.append(commit.id)
         # invalidate parsed refs after commit
         self.repository._parsed_refs = self.repository._get_parsed_refs()
+        # Update vcs repository object & recreate dulwich repo
+        self.repository.invalidate_revisions()
         tip = self.repository.get_changeset()
         self.reset()
         return tip

vcs/backends/git/repository.py

@@ -43,8 +43,8 @@ class GitRepository(BaseRepository):
     scm = 'git'
 
     def __init__(self, repo_path, create=False, src_url=None,
-                 update_after_clone=False, bare=False):
-
+        update_after_clone=False, bare=False, use_revisions_cache=False):
+        self.use_revisions_cache = use_revisions_cache
         self.path = abspath(repo_path)
         repo = self._get_repo(create, src_url, update_after_clone, bare)
         self.bare = repo.bare
@@ -68,14 +68,6 @@ def head(self):
         except KeyError:
             return None
 
-    @LazyProperty
-    def revisions(self):
-        """
-        Returns list of revisions' ids, in ascending order.  Being lazy
-        attribute allows external tools to inject shas from cache.
-        """
-        return self._get_all_revisions()
-
     @classmethod
     def _run_git_command(cls, cmd, **opts):
         """

vcs/backends/hg/inmemory.py

@@ -4,7 +4,7 @@
 from vcs.backends.base import BaseInMemoryChangeset
 from vcs.exceptions import RepositoryError
 
-from vcs.utils.hgcompat import memfilectx, memctx, hex, tolocal
+from vcs.utils.hgcompat import memfilectx, memctx, tolocal
 
 
 class MercurialInMemoryChangeset(BaseInMemoryChangeset):
@@ -98,16 +98,17 @@ def filectxfn(_repo, memctx, path):
         commit_ctx._date = date
 
         # TODO: Catch exceptions!
-        n = self.repository._repo.commitctx(commit_ctx)
+        self.repository._repo.commitctx(commit_ctx)
         # Returns mercurial node
         self._commit_ctx = commit_ctx  # For reference
         # Update vcs repository object & recreate mercurial _repo
         # new_ctx = self.repository._repo[node]
         # new_tip = self.repository.get_changeset(new_ctx.hex())
-        new_id = hex(n)
-        self.repository.revisions.append(new_id)
         self._repo = self.repository._get_repo(create=False)
+        self.repository.invalidate_revisions()
         self.repository.branches = self.repository._get_branches()
         tip = self.repository.get_changeset()
         self.reset()
         return tip
+
+# invalidate

vcs/backends/hg/repository.py

@@ -47,7 +47,7 @@ class MercurialRepository(BaseRepository):
     scm = 'hg'
 
     def __init__(self, repo_path, create=False, baseui=None, src_url=None,
-                 update_after_clone=False):
+            update_after_clone=False, use_revisions_cache=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
@@ -59,13 +59,16 @@ def __init__(self, repo_path, create=False, baseui=None, src_url=None,
         :param src_url=None: would try to clone repository from given location
         :param update_after_clone=False: sets update of working copy after
           making a clone
+        :param use_revisions_cache: if set to True, would try to use cached
+          revisions list (and saves it if cache does not exist).
         """
 
         if not isinstance(repo_path, str):
             raise VCSError('Mercurial backend requires repository path to '
                            'be instance of <str> got %s instead' %
                            type(repo_path))
 
+        self.use_revisions_cache = use_revisions_cache
         self.path = abspath(repo_path)
         self.baseui = baseui or ui.ui()
         # We've set path and ui, now we can set _repo itself
@@ -80,14 +83,6 @@ def _empty(self):
         # return len(self._repo.changelog) == 0
         return len(self.revisions) == 0
 
-    @LazyProperty
-    def revisions(self):
-        """
-        Returns list of revisions' ids, in ascending order.  Being lazy
-        attribute allows external tools to inject shas from cache.
-        """
-        return self._get_all_revisions()
-
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
@@ -549,7 +544,7 @@ def get_user_name(self, config_file=None):
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
-        username = self.get_config_value('ui', 'username')
+        username = self.get_config_value('ui', 'username', config_file)
         if username:
             return author_name(username)
         return None
@@ -561,7 +556,7 @@ def get_user_email(self, config_file=None):
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
-        username = self.get_config_value('ui', 'username')
+        username = self.get_config_value('ui', 'username', config_file)
         if username:
             return author_email(username)
         return None

vcs/tests/conf.py

@@ -58,5 +58,6 @@ def get_new_dir(title):
 PACKAGE_DIR = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..'))
 _dest = jn(TEST_TMP_PATH, 'aconfig')
-shutil.copy(jn(THIS, 'aconfig'), _dest)
+TEST_USER_CONFIG_FILE_SRC = jn(THIS, 'aconfig')
+shutil.copy(TEST_USER_CONFIG_FILE_SRC, _dest)
 TEST_USER_CONFIG_FILE = _dest

vcs/tests/test_repository.py

@@ -1,34 +1,37 @@
 from __future__ import with_statement
+import os
+import gzip
 import datetime
+from mock import Mock
 from vcs.tests.base import BackendTestMixin
 from vcs.tests.conf import SCM_TESTS
-from vcs.tests.conf import TEST_USER_CONFIG_FILE
+from vcs.tests.conf import TEST_USER_CONFIG_FILE_SRC
 from vcs.nodes import FileNode
 from vcs.utils.compat import unittest
 from vcs.exceptions import ChangesetDoesNotExistError
 
 
 class RepositoryBaseTest(BackendTestMixin):
-    recreate_repo_per_test = False
+    recreate_repo_per_test = True
 
     @classmethod
     def _get_commits(cls):
         return super(RepositoryBaseTest, cls)._get_commits()[:1]
 
     def test_get_config_value(self):
         self.assertEqual(self.repo.get_config_value('universal', 'foo',
-            TEST_USER_CONFIG_FILE), 'bar')
+            TEST_USER_CONFIG_FILE_SRC), 'bar')
 
     def test_get_config_value_defaults_to_None(self):
         self.assertEqual(self.repo.get_config_value('universal', 'nonexist',
-            TEST_USER_CONFIG_FILE), None)
+            TEST_USER_CONFIG_FILE_SRC), None)
 
     def test_get_user_name(self):
-        self.assertEqual(self.repo.get_user_name(TEST_USER_CONFIG_FILE),
+        self.assertEqual(self.repo.get_user_name(TEST_USER_CONFIG_FILE_SRC),
             'Foo Bar')
 
     def test_get_user_email(self):
-        self.assertEqual(self.repo.get_user_email(TEST_USER_CONFIG_FILE),
+        self.assertEqual(self.repo.get_user_email(TEST_USER_CONFIG_FILE_SRC),
             'foo.bar@example.com')
 
     def test_repo_equality(self):
@@ -45,6 +48,58 @@ class dummy(object):
             path = self.repo.path
         self.assertTrue(self.repo != dummy())
 
+    def test_repo_invalidate_revisions(self):
+        revisions = self.repo.revisions[:] # copy
+        # at least in one test make sure revisions list is not empty
+        self.assertTrue(len(revisions) > 0)
+        self.repo.revisions = 'this should be recreated anyway'
+        self.repo.invalidate_revisions()
+        self.assertEqual(self.repo.revisions, revisions)
+
+    def test_repo_invalidate_revisions_itself_does_not_access_revisions(self):
+        self.repo._get_all_revisions = Mock()
+        self.repo.invalidate_revisions()
+        self.assertFalse(self.repo._get_all_revisions.called)
+        self.repo.revisions # access attribute
+        self.assertTrue(self.repo._get_all_revisions.called)
+
+    def test_repo_respects_use_revisions_cache(self):
+        revisions = self.repo.revisions[:] # copy
+        self.repo.use_revisions_cache = True
+        self.repo.invalidate_revisions()
+        self.repo.revisions # access attribute
+        cached = gzip.open(self.repo.get_revisions_cache_path()).read().splitlines()
+        self.assertEqual(revisions, cached)
+
+    def test_get_cached_revisions(self):
+        self.repo.cache_revisions()
+        cache_path = self.repo.get_revisions_cache_path()
+        try:
+            fout = gzip.open(cache_path, 'w')
+            fout.write('foo\nbar')
+        finally:
+            fout.close()
+
+        self.assertEqual(self.repo.get_cached_revisions(), ['foo', 'bar'])
+
+    def test_cache_revisions(self):
+        revisions = self.repo.revisions[:] # copy
+        self.repo.cache_revisions()
+        cache_path = self.repo.get_revisions_cache_path()
+        cached_revisions = gzip.open(cache_path).read().splitlines()
+        self.assertEqual(revisions, cached_revisions)
+
+    def test_repo_invalidate_recreates_cache(self):
+        self.repo.use_revisions_cache = True
+        self.repo.invalidate_revisions()
+        self.repo.revisions # access attribute
+        revisions = self.repo.revisions[:] # copy
+        os.remove(self.repo.get_revisions_cache_path())
+        self.repo.invalidate_revisions()
+        self.repo.revisions # access attribute
+        cached = gzip.open(self.repo.get_revisions_cache_path()).read().splitlines()
+        self.assertEqual(revisions, cached)
+
 
 class RepositoryGetDiffTest(BackendTestMixin):