From 15b8c346f5fac8fb8bdca2d2fa8c09ea472cc1d0 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 3 Apr 2024 13:34:32 +0200 Subject: [PATCH 1/9] Upgrade requirements --- requirements.in | 33 ++++++++++++++++----------------- requirements.txt | 40 +++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/requirements.in b/requirements.in index 915baa62d90..bd90411fdcb 100644 --- a/requirements.in +++ b/requirements.in @@ -1,39 +1,38 @@ # The file contains the direct ckan requirements (python3). # Use pip-compile to create a requirements.txt file from this -alembic==1.13.0 -Babel==2.13.1 +alembic==1.13.1 +Babel==2.14.0 Beaker==1.12.1 bleach==6.1.0 blinker==1.7.0 certifi>=2023.7.22 click==8.1.7 -dominate==2.9.0 -feedgen==0.9.0 -Flask==3.0.0 +dominate==2.9.1 +feedgen==1.0.0 +Flask==3.0.2 Flask-Babel==4.0.0 Flask-Login==0.6.3 Flask-WTF==1.2.1 -greenlet>=2.0.2 -Jinja2==3.1.2 -Markdown==3.5.1 +Jinja2==3.1.3 +Markdown==3.6 passlib==1.7.4 polib==1.2.0 -psycopg2==2.9.7 +psycopg2==2.9.9 PyJWT==2.8.0 -pyparsing==3.0.7 +pyparsing==3.1.2 python-magic==0.4.27 pysolr==3.9.0 -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 pytz PyUtilib==6.0.0 pyyaml==6.0.1 requests==2.31.0 -rq==1.15.1 +rq==1.16.1 simplejson==3.19.2 -SQLAlchemy[mypy]==1.4.49 +SQLAlchemy[mypy]==1.4.52 sqlparse==0.4.4 -typing_extensions==4.8.0 -tzlocal==5.0.1 +typing_extensions==4.10.0 +tzlocal==5.2 webassets==2.0 -Werkzeug[watchdog]==3.0.1 -zope.interface==6.0 +Werkzeug[watchdog]==3.0.2 +zope.interface==6.2 diff --git a/requirements.txt b/requirements.txt index c61751543c1..91ad4d7e1f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile requirements.in # -alembic==1.13.0 +alembic==1.13.1 # via -r requirements.in -babel==2.13.1 +babel==2.14.0 # via # -r requirements.in # flask-babel @@ -31,11 +31,11 @@ click==8.1.7 # rq deprecated==1.2.13 # via redis -dominate==2.9.0 +dominate==2.9.1 # via -r requirements.in -feedgen==0.9.0 +feedgen==1.0.0 # via -r requirements.in -flask==3.0.0 +flask==3.0.2 # via # -r requirements.in # flask-babel @@ -48,9 +48,7 @@ flask-login==0.6.3 flask-wtf==1.2.1 # via -r requirements.in greenlet==2.0.2 - # via - # -r requirements.in - # sqlalchemy + # via sqlalchemy idna==3.3 # via requests importlib-metadata==6.3.0 @@ -61,7 +59,7 @@ itsdangerous==2.1.2 # via # flask # flask-wtf -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements.in # flask @@ -70,7 +68,7 @@ lxml==4.9.1 # via feedgen mako==1.2.2 # via alembic -markdown==3.5.1 +markdown==3.6 # via -r requirements.in markupsafe==2.1.1 # via @@ -90,17 +88,17 @@ passlib==1.7.4 # via -r requirements.in polib==1.2.0 # via -r requirements.in -psycopg2==2.9.7 +psycopg2==2.9.9 # via -r requirements.in pyjwt==2.8.0 # via -r requirements.in -pyparsing==3.0.7 +pyparsing==3.1.2 # via - # -r requirements.in - # packaging + # -r requirements.in + # packaging pysolr==3.9.0 # via -r requirements.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements.in # feedgen @@ -120,7 +118,7 @@ requests==2.31.0 # via # -r requirements.in # pysolr -rq==1.15.1 +rq==1.16.1 # via -r requirements.in simplejson==3.19.2 # via -r requirements.in @@ -129,7 +127,7 @@ six==1.16.0 # bleach # python-dateutil # pyutilib -sqlalchemy[mypy]==1.4.49 +sqlalchemy[mypy]==1.4.52 # via # -r requirements.in # alembic @@ -140,13 +138,13 @@ sqlparse==0.4.4 # via -r requirements.in tomli==2.0.1 # via mypy -typing-extensions==4.8.0 +typing-extensions==4.10.0 # via # -r requirements.in # alembic # mypy # sqlalchemy2-stubs -tzlocal==5.0.1 +tzlocal==5.2 # via -r requirements.in urllib3==1.26.9 # via requests @@ -156,7 +154,7 @@ webassets==2.0 # via -r requirements.in webencodings==0.5.1 # via bleach -werkzeug[watchdog]==3.0.1 +werkzeug[watchdog]==3.0.2 # via # -r requirements.in # flask @@ -168,7 +166,7 @@ wtforms==3.0.1 # via flask-wtf zipp==3.15.0 # via importlib-metadata -zope-interface==6.0 +zope-interface==6.2 # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: From b07da9103675334e4f5f93c48c171e0317a5c1b8 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 3 Apr 2024 13:36:00 +0200 Subject: [PATCH 2/9] Upgrade requirements' requirements --- requirements.txt | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/requirements.txt b/requirements.txt index 91ad4d7e1f5..d05fee58e1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,8 @@ # alembic==1.13.1 # via -r requirements.in +async-timeout==4.0.3 + # via redis babel==2.14.0 # via # -r requirements.in @@ -18,19 +20,17 @@ blinker==1.7.0 # via # -r requirements.in # flask -certifi==2023.7.22 +certifi==2024.2.2 # via # -r requirements.in # requests -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via # -r requirements.in # flask # rq -deprecated==1.2.13 - # via redis dominate==2.9.1 # via -r requirements.in feedgen==1.0.0 @@ -47,11 +47,11 @@ flask-login==0.6.3 # via -r requirements.in flask-wtf==1.2.1 # via -r requirements.in -greenlet==2.0.2 +greenlet==3.0.3 # via sqlalchemy -idna==3.3 +idna==3.6 # via requests -importlib-metadata==6.3.0 +importlib-metadata==7.1.0 # via # flask # markdown @@ -64,26 +64,24 @@ jinja2==3.1.3 # -r requirements.in # flask # flask-babel -lxml==4.9.1 +lxml==5.2.1 # via feedgen -mako==1.2.2 +mako==1.3.2 # via alembic markdown==3.6 # via -r requirements.in -markupsafe==2.1.1 +markupsafe==2.1.5 # via # jinja2 # mako # werkzeug # wtforms -mypy==0.971 +mypy==1.9.0 # via sqlalchemy -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via mypy nose==1.3.7 # via pyutilib -packaging==21.3 - # via redis passlib==1.7.4 # via -r requirements.in polib==1.2.0 @@ -93,9 +91,7 @@ psycopg2==2.9.9 pyjwt==2.8.0 # via -r requirements.in pyparsing==3.1.2 - # via - # -r requirements.in - # packaging + # via -r requirements.in pysolr==3.9.0 # via -r requirements.in python-dateutil==2.9.0.post0 @@ -104,7 +100,7 @@ python-dateutil==2.9.0.post0 # feedgen python-magic==0.4.27 # via -r requirements.in -pytz==2022.7.1 +pytz==2024.1 # via # -r requirements.in # flask-babel @@ -112,7 +108,7 @@ pyutilib==6.0.0 # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in -redis==4.1.4 +redis==5.0.3 # via rq requests==2.31.0 # via @@ -132,7 +128,7 @@ sqlalchemy[mypy]==1.4.52 # -r requirements.in # alembic # sqlalchemy -sqlalchemy2-stubs==0.0.2a36 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy sqlparse==0.4.4 # via -r requirements.in @@ -146,9 +142,9 @@ typing-extensions==4.10.0 # sqlalchemy2-stubs tzlocal==5.2 # via -r requirements.in -urllib3==1.26.9 +urllib3==2.2.1 # via requests -watchdog==3.0.0 +watchdog==4.0.0 # via werkzeug webassets==2.0 # via -r requirements.in @@ -160,11 +156,9 @@ werkzeug[watchdog]==3.0.2 # flask # flask-login # werkzeug -wrapt==1.14.0 - # via deprecated -wtforms==3.0.1 +wtforms==3.1.2 # via flask-wtf -zipp==3.15.0 +zipp==3.18.1 # via importlib-metadata zope-interface==6.2 # via -r requirements.in From e70147932df544c4c4e4845362371a4f3d0f7324 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 3 Apr 2024 13:42:46 +0200 Subject: [PATCH 3/9] Upgrade dev requirements --- dev-requirements.txt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index ee357d5db4f..6226fe0ce88 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,24 +1,24 @@ # These are packages that are required by ckan developers - for running ckan in debug mode, running ckan tests, building the docs and to pip-compile the requirements.in file. -beautifulsoup4==4.12.2 -cookiecutter==2.5.0 +beautifulsoup4==4.12.3 +cookiecutter==2.6.0 coveralls #Let Unpinned - Requires latest coveralls -Faker==20.1.0 +Faker==24.4.0 factory-boy==3.3.0 -flask-debugtoolbar==0.14.0 -freezegun==1.3.1 +flask-debugtoolbar==0.14.1 +freezegun==1.4.0 ipdb==0.13.13 -pip-tools==7.3.0 -Pillow==10.1.0 -responses==0.24.1 +pip-tools==7.4.1 +Pillow==10.3.0 +responses==0.25.0 sphinx-rtd-theme==2.0.0 sphinx==7.2.6 toml==0.10.2 towncrier==23.11.0 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-factoryboy==2.6.0 +pytest==8.1.1 +pytest-cov==5.0.0 +pytest-factoryboy==2.7.0 pytest-freezegun==0.4.2 -pytest-rerunfailures==13.0 -pytest-split==0.8.1 +pytest-rerunfailures==14.0 +pytest-split==0.8.2 From df5353e38a810f100d4e4f8b2bd4de8c6e4d7d48 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 5 Apr 2024 11:01:29 +0200 Subject: [PATCH 4/9] Remove setup method in queues test class Moved the cleanup code to a fixture --- ckan/tests/helpers.py | 13 +--------- ckan/tests/pytest_ckan/fixtures.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 7b3fdc538e3..addbf97cb39 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -350,23 +350,12 @@ def teardown_class(cls): config.update(cls._original_config) -@pytest.mark.usefixtures("with_test_worker") +@pytest.mark.usefixtures("with_test_worker", "clean_queues") class RQTestBase(object): """ Base class for tests of RQ functionality. """ - def setup(self): - u""" - Delete all RQ queues and jobs. - """ - # See https://github.com/nvie/rq/issues/731 - redis_conn = connect_to_redis() - for queue in rq.Queue.all(connection=redis_conn): - queue.empty() - redis_conn.srem(rq.Queue.redis_queues_keys, queue._key) - redis_conn.delete(queue._key) - def all_jobs(self): u""" Get a list of all RQ jobs. diff --git a/ckan/tests/pytest_ckan/fixtures.py b/ckan/tests/pytest_ckan/fixtures.py index c9a529a0036..1923ee3f7c8 100644 --- a/ckan/tests/pytest_ckan/fixtures.py +++ b/ckan/tests/pytest_ckan/fixtures.py @@ -216,6 +216,23 @@ def reset_index(): return search.clear_all +def _empty_queues(): + + conn = redis.connect_to_redis() + for queue in rq.Queue.all(connection=conn): + queue.empty() + queue.delete() + + +@pytest.fixture(scope=u"session") +def reset_queues(): + """Callable for emptying and deleting the queues. + + If possible use the ``clean_queues`` fixture instead. + """ + return _empty_queues + + @pytest.fixture(scope="session") def reset_redis(): """Callable for removing all keys from Redis. @@ -307,6 +324,28 @@ def test_example(self): reset_db() +@pytest.fixture +def clean_queues(reset_queues): + """Empties and deleted all queues. + + This can be used either for all tests in a class:: + + @pytest.mark.usefixtures("clean_queues") + class TestExample(object): + + def test_example(self): + + or for a single test:: + + class TestExample(object): + + @pytest.mark.usefixtures("clean_queues") + def test_example(self): + + """ + reset_queues() + + @pytest.fixture(scope="session") def migrate_db_for(): """Apply database migration defined by plugin. From e3a4bfa6c6f3b47e73b0010d5f7fc23336111c4a Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 5 Apr 2024 11:02:24 +0200 Subject: [PATCH 5/9] Replace setup methods with fixtures --- ckan/tests/logic/auth/test_create.py | 78 ++++++++++----------- ckan/tests/logic/auth/test_get.py | 101 ++++++++++++++------------- 2 files changed, 91 insertions(+), 88 deletions(-) diff --git a/ckan/tests/logic/auth/test_create.py b/ckan/tests/logic/auth/test_create.py index 46d34bf71d2..608784602d3 100644 --- a/ckan/tests/logic/auth/test_create.py +++ b/ckan/tests/logic/auth/test_create.py @@ -405,6 +405,24 @@ def test_auth_user_is_allowed_to_create_tokens(self): user=user[u"name"], ) +@pytest.fixture +def members_fixtures(): + org_admin = factories.User() + org_editor = factories.User() + org_member = factories.User() + + org = factories.Organization( + users=[ + {'name': org_admin['name'], 'capacity': 'admin'}, + {'name': org_editor['name'], 'capacity': 'editor'}, + {'name': org_member['name'], 'capacity': 'member'}, + ] + ) + + dataset = factories.Dataset(owner_org=org['id']) + + return locals() + @pytest.mark.usefixtures("non_clean_db") @pytest.mark.ckan_config(u"ckan.auth.allow_dataset_collaborators", True) @@ -416,47 +434,29 @@ def _get_context(self, user): "user": user if isinstance(user, str) else user.get("name"), } - def setup(self): - - self.org_admin = factories.User() - self.org_editor = factories.User() - self.org_member = factories.User() - - self.normal_user = factories.User() - - self.org = factories.Organization( - users=[ - {'name': self.org_admin['name'], 'capacity': 'admin'}, - {'name': self.org_editor['name'], 'capacity': 'editor'}, - {'name': self.org_member['name'], 'capacity': 'member'}, - ] - ) - - self.dataset = factories.Dataset(owner_org=self.org['id']) - - def test_create_org_admin_is_authorized(self): + def test_create_org_admin_is_authorized(self, members_fixtures): - context = self._get_context(self.org_admin) + context = self._get_context(members_fixtures["org_admin"]) assert helpers.call_auth( - 'package_collaborator_create', context=context, id=self.dataset['id']) + 'package_collaborator_create', context=context, id=members_fixtures["dataset"]['id']) - def test_create_org_editor_is_not_authorized(self): + def test_create_org_editor_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_editor) + context = self._get_context(members_fixtures["org_editor"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_create', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_create_org_member_is_not_authorized(self): + def test_create_org_member_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_member) + context = self._get_context(members_fixtures["org_member"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_create', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_create_non_org_user_is_not_authorized(self): + def test_create_non_org_user_is_not_authorized(self, members_fixtures): user = factories.User() @@ -464,9 +464,9 @@ def test_create_non_org_user_is_not_authorized(self): with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_create', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_create_org_admin_from_other_org_is_not_authorized(self): + def test_create_org_admin_from_other_org_is_not_authorized(self, members_fixtures): org_admin2 = factories.User() factories.Organization( @@ -479,44 +479,44 @@ def test_create_org_admin_from_other_org_is_not_authorized(self): with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_create', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_create_missing_org_is_not_authorized(self): + def test_create_missing_org_is_not_authorized(self, members_fixtures): dataset = factories.Dataset(owner_org=None) - context = self._get_context(self.org_admin) + context = self._get_context(members_fixtures["org_admin"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_create', context=context, id=dataset['id']) @pytest.mark.ckan_config('ckan.auth.allow_admin_collaborators', True) - def test_create_collaborator_admin_is_authorized(self): + def test_create_collaborator_admin_is_authorized(self, members_fixtures): user = factories.User() helpers.call_action( 'package_collaborator_create', - id=self.dataset['id'], user_id=user['id'], capacity='admin') + id=members_fixtures["dataset"]['id'], user_id=user['id'], capacity='admin') context = self._get_context(user) assert helpers.call_auth( - 'package_collaborator_create', context=context, id=self.dataset['id']) + 'package_collaborator_create', context=context, id=members_fixtures["dataset"]['id']) @pytest.mark.parametrize('role', ['editor', 'member']) - def test_create_collaborator_editor_and_member_are_not_authorized(self, role): + def test_create_collaborator_editor_and_member_are_not_authorized(self, role, members_fixtures): user = factories.User() helpers.call_action( 'package_collaborator_create', - id=self.dataset['id'], user_id=user['id'], capacity=role) + id=members_fixtures["dataset"]['id'], user_id=user['id'], capacity=role) context = self._get_context(user) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_create', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) @pytest.mark.ckan_config('ckan.auth.create_dataset_if_not_in_organization', True) @pytest.mark.ckan_config('ckan.auth.create_unowned_dataset', True) diff --git a/ckan/tests/logic/auth/test_get.py b/ckan/tests/logic/auth/test_get.py index 29c97b40e2f..7e38762f8ad 100644 --- a/ckan/tests/logic/auth/test_get.py +++ b/ckan/tests/logic/auth/test_get.py @@ -383,6 +383,27 @@ def test_resource_view_show_private_member(self): context=context, id=resource_view['id']) +@pytest.fixture +def members_fixtures(): + org_admin = factories.User() + org_editor = factories.User() + org_member = factories.User() + + normal_user = factories.User() + + org = factories.Organization( + users=[ + {'name': org_admin['name'], 'capacity': 'admin'}, + {'name': org_editor['name'], 'capacity': 'editor'}, + {'name': org_member['name'], 'capacity': 'member'}, + ] + ) + + dataset = factories.Dataset(owner_org=org['id']) + + return locals() + + @pytest.mark.usefixtures("non_clean_db") @pytest.mark.ckan_config(u"ckan.auth.allow_dataset_collaborators", True) class TestPackageMemberList(object): @@ -394,48 +415,30 @@ def _get_context(self, user): 'user': user if isinstance(user, str) else user.get('name') } - def setup(self): - - self.org_admin = factories.User() - self.org_editor = factories.User() - self.org_member = factories.User() - - self.normal_user = factories.User() - - self.org = factories.Organization( - users=[ - {'name': self.org_admin['name'], 'capacity': 'admin'}, - {'name': self.org_editor['name'], 'capacity': 'editor'}, - {'name': self.org_member['name'], 'capacity': 'member'}, - ] - ) - - self.dataset = factories.Dataset(owner_org=self.org['id']) - - def test_list_org_admin_is_authorized(self): + def test_list_org_admin_is_authorized(self, members_fixtures): - context = self._get_context(self.org_admin) + context = self._get_context(members_fixtures["org_admin"]) assert helpers.call_auth( 'package_collaborator_list', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_list_org_editor_is_not_authorized(self): + def test_list_org_editor_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_editor) + context = self._get_context(members_fixtures["org_editor"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_list_org_member_is_not_authorized(self): + def test_list_org_member_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_member) + context = self._get_context(members_fixtures["org_member"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_list_org_admin_from_other_org_is_not_authorized(self): + def test_list_org_admin_from_other_org_is_not_authorized(self, members_fixtures): org_admin2 = factories.User() factories.Organization( users=[ @@ -447,67 +450,67 @@ def test_list_org_admin_from_other_org_is_not_authorized(self): with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list_for_user', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) @pytest.mark.ckan_config('ckan.auth.allow_admin_collaborators', True) - def test_list_collaborator_admin_is_authorized(self): + def test_list_collaborator_admin_is_authorized(self, members_fixtures): user = factories.User() helpers.call_action( 'package_collaborator_create', - id=self.dataset['id'], user_id=user['id'], capacity='admin') + id=members_fixtures["dataset"]['id'], user_id=user['id'], capacity='admin') context = self._get_context(user) assert helpers.call_auth( - 'package_collaborator_list', context=context, id=self.dataset['id']) + 'package_collaborator_list', context=context, id=members_fixtures["dataset"]['id']) @pytest.mark.parametrize('role', ['editor', 'member']) - def test_list_collaborator_editor_and_member_are_not_authorized(self, role): + def test_list_collaborator_editor_and_member_are_not_authorized(self, role, members_fixtures): user = factories.User() helpers.call_action( 'package_collaborator_create', - id=self.dataset['id'], user_id=user['id'], capacity=role) + id=members_fixtures["dataset"]['id'], user_id=user['id'], capacity=role) context = self._get_context(user) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list', - context=context, id=self.dataset['id']) + context=context, id=members_fixtures["dataset"]['id']) - def test_user_list_own_user_is_authorized(self): + def test_user_list_own_user_is_authorized(self, members_fixtures): - context = self._get_context(self.normal_user) + context = self._get_context(members_fixtures["normal_user"]) assert helpers.call_auth( 'package_collaborator_list_for_user', - context=context, id=self.normal_user['id']) + context=context, id=members_fixtures["normal_user"]['id']) - def test_user_list_org_admin_is_not_authorized(self): + def test_user_list_org_admin_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_admin) + context = self._get_context(members_fixtures["org_admin"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list_for_user', - context=context, id=self.normal_user['id']) + context=context, id=members_fixtures["normal_user"]['id']) - def test_user_list_org_editor_is_not_authorized(self): + def test_user_list_org_editor_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_editor) + context = self._get_context(members_fixtures["org_editor"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list_for_user', - context=context, id=self.normal_user['id']) + context=context, id=members_fixtures["normal_user"]['id']) - def test_user_list_org_member_is_not_authorized(self): + def test_user_list_org_member_is_not_authorized(self, members_fixtures): - context = self._get_context(self.org_member) + context = self._get_context(members_fixtures["org_member"]) with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list_for_user', - context=context, id=self.normal_user['id']) + context=context, id=members_fixtures["normal_user"]['id']) - def test_user_list_org_admin_from_other_org_is_not_authorized(self): + def test_user_list_org_admin_from_other_org_is_not_authorized(self, members_fixtures): org_admin2 = factories.User() factories.Organization( users=[ @@ -519,7 +522,7 @@ def test_user_list_org_admin_from_other_org_is_not_authorized(self): with pytest.raises(logic.NotAuthorized): helpers.call_auth( 'package_collaborator_list_for_user', - context=context, id=self.normal_user['id']) + context=context, id=members_fixtures["normal_user"]['id']) @pytest.mark.ckan_config('ckan.auth.create_dataset_if_not_in_organization', True) @pytest.mark.ckan_config('ckan.auth.create_unowned_dataset', True) From 65d887c87f122b4bd0a1fe5dcb7ffcba1b74727e Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 5 Apr 2024 11:21:12 +0200 Subject: [PATCH 6/9] Typing --- ckan/lib/app_globals.py | 2 -- ckan/lib/jobs.py | 2 +- ckan/lib/redis.py | 10 +++++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 3dfdef193ed..7851fa0a0d7 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -210,8 +210,6 @@ def _init(self): self.ckan_version = ckan.__version__ version = parse_version(self.ckan_version) - if not isinstance(version, Version): - raise ValueError(self.ckan_version) self.ckan_base_version = version.base_version if not version.is_prerelease: diff --git a/ckan/lib/jobs.py b/ckan/lib/jobs.py index 641df60a470..f081e72e569 100644 --- a/ckan/lib/jobs.py +++ b/ckan/lib/jobs.py @@ -43,7 +43,7 @@ _queues: dict[str, rq.Queue] = {} -def _connect() -> Redis: # type: ignore +def _connect() -> Redis: u''' Connect to Redis and tell RQ about it. diff --git a/ckan/lib/redis.py b/ckan/lib/redis.py index c976f4438f9..f90fc4acd60 100644 --- a/ckan/lib/redis.py +++ b/ckan/lib/redis.py @@ -1,6 +1,6 @@ # encoding: utf-8 -u''' +''' Redis utilities. .. versionadded:: 2.7 @@ -21,8 +21,8 @@ _connection_pool = None -def connect_to_redis() -> Redis: # type: ignore - u''' +def connect_to_redis() -> Redis: + ''' (Lazily) connect to Redis. The connection is set up but not actually established. The latter @@ -42,7 +42,7 @@ def connect_to_redis() -> Redis: # type: ignore def is_redis_available() -> bool: - u''' + ''' Check whether Redis is available. :returns: The availability of Redis. @@ -52,7 +52,7 @@ def is_redis_available() -> bool: ''' redis_conn = connect_to_redis() try: - return redis_conn.ping() + return redis_conn.ping() is True except Exception: log.exception(u'Redis is not available') return False From b6d9eec151c2a3a8038cc68fcd42d9f241685d2d Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 5 Apr 2024 12:58:02 +0200 Subject: [PATCH 7/9] lint, types --- ckan/lib/app_globals.py | 2 +- ckan/tests/logic/auth/test_create.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 7851fa0a0d7..0cf0649777a 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -6,7 +6,7 @@ import logging from threading import Lock from typing import Any, Union -from packaging.version import parse as parse_version, Version +from packaging.version import parse as parse_version import ckan import ckan.model as model diff --git a/ckan/tests/logic/auth/test_create.py b/ckan/tests/logic/auth/test_create.py index 608784602d3..6c9c2283d2a 100644 --- a/ckan/tests/logic/auth/test_create.py +++ b/ckan/tests/logic/auth/test_create.py @@ -405,6 +405,7 @@ def test_auth_user_is_allowed_to_create_tokens(self): user=user[u"name"], ) + @pytest.fixture def members_fixtures(): org_admin = factories.User() From 135267470c6735abeb827d290a0af29c2e7e5db2 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 5 Apr 2024 13:27:07 +0200 Subject: [PATCH 8/9] Fix version handling in docs configuration Remove completely our own version parsing and rely on the one from packaging. Older versions of packaging used to be more lenient with our release tags ("ckan-x.y.z") but now they will fail. We need to remove the `ckan-` bit before parsing. --- doc/conf.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index eca46d5929f..d00bae546bc 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -142,23 +142,10 @@ def get_release_tags(): # git tag -l prints out the tags in the right order anyway, but don't rely # on that, sort them again here for good measure. - release_tags_.sort(key=version_parse) + release_tags_ = sorted(release_tags_, key=lambda tag: version_parse(tag.lstrip('ckan-'))) return release_tags_ -def parse_version(version_): - '''Parses version string - ckan-2.1.3 -> ('2', '1', '3') - ckan-2.1 -> ('2', '1', None) (the occasion when we didn't do semver) - ''' - global version_re - if version_re is None: - version_re = re.compile('(?:ckan-)?(\d+)\.(\d+)(?:\.(\d+))?[a-z]?') - if isinstance(version_, bytes): - version_ = version_.decode() - return version_re.match(version_).groups() - - def get_equivalent_point_release(version_): '''Returns the equivalent point release of any given version. @@ -166,7 +153,8 @@ def get_equivalent_point_release(version_): ckan-2.1.3 -> ckan-2.1 ckan-2.1 -> ckan-2.1 (the occasion when we didn't do semver) ''' - return 'ckan-%s.%s' % parse_version(version_)[:2] + version = version_parse(version_.lstrip("ckan-")) + return f"ckan-{version.major}.{version.minor}" def get_point_releases(): @@ -262,9 +250,9 @@ def get_previous_release_version() -> str: eg if the latest release is 2.9.5, it returns 2.8.10 """ - current_version = parse_version(get_current_release_version()) + current_version = version_parse(get_current_release_version()) - previous_tag_prefix = f"ckan-{current_version[0]}.{int(current_version[1]) - 1}" + previous_tag_prefix = f"ckan-{current_version.major}.{current_version.minor - 1}" previous_version_tags = [ r for r in get_release_tags() if r.startswith(previous_tag_prefix) From 239f91b607ecef87ce24d2a79841dd9766d88730 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 Apr 2024 11:44:30 +0200 Subject: [PATCH 9/9] Add packaging as first level requirement --- requirements.in | 1 + requirements.txt | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index bd90411fdcb..18977bb094b 100644 --- a/requirements.in +++ b/requirements.in @@ -15,6 +15,7 @@ Flask-Login==0.6.3 Flask-WTF==1.2.1 Jinja2==3.1.3 Markdown==3.6 +packaging==24.0 passlib==1.7.4 polib==1.2.0 psycopg2==2.9.9 diff --git a/requirements.txt b/requirements.txt index d05fee58e1a..59ff7f2d99c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,13 +49,13 @@ flask-wtf==1.2.1 # via -r requirements.in greenlet==3.0.3 # via sqlalchemy -idna==3.6 +idna==3.7 # via requests importlib-metadata==7.1.0 # via # flask # markdown -itsdangerous==2.1.2 +itsdangerous==2.2.0 # via # flask # flask-wtf @@ -66,7 +66,7 @@ jinja2==3.1.3 # flask-babel lxml==5.2.1 # via feedgen -mako==1.3.2 +mako==1.3.3 # via alembic markdown==3.6 # via -r requirements.in @@ -82,6 +82,8 @@ mypy-extensions==1.0.0 # via mypy nose==1.3.7 # via pyutilib +packaging==24.0 + # via -r requirements.in passlib==1.7.4 # via -r requirements.in polib==1.2.0 @@ -108,7 +110,7 @@ pyutilib==6.0.0 # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in -redis==5.0.3 +redis==5.0.4 # via rq requests==2.31.0 # via