Merge branch 'master' into 6118-table-designer

changes/4193.feature

@@ -0,0 +1 @@
+Allow sysadmins to change usernames of other accounts

changes/8121.feature

@@ -0,0 +1 @@
+Add reCAPTCHA protection on login and password reset

changes/8130.bugfix

@@ -0,0 +1 @@
+``use_default_schema`` in ``package_show`` is now evaluated as boolean.

changes/8138.bugfix

@@ -0,0 +1 @@
+Allow using ``.`` in Solr local parser parameters

changes/8141.bugfix

@@ -0,0 +1 @@
+Hide invite user form if the user can't create users

changes/8148.bugfix

@@ -0,0 +1 @@
+Add error notification when rebuilding the search index via the cli when the requested package can't be found.

changes/8154.feature

@@ -0,0 +1 @@
+Resource view list items now have an additional `view-item` class.

changes/8179.bugfix

@@ -0,0 +1 @@
+Correct package_patch docstring re: updating resources

ckan/cli/search_index.py

@@ -48,6 +48,8 @@ def rebuild(
                 defer_commit=(not commit_each),
                 quiet=quiet and not verbose,
                 clear=clear)
+    except logic.NotFound:
+        error_shout("Couldn't find package %s" % package_id)
     except Exception as e:
         error_shout(e)
     if not commit_each:

ckan/lib/app_globals.py

@@ -6,7 +6,7 @@
 import logging
 from threading import Lock
 from typing import Any, Union
-from packaging.version import parse as parse_version, Version
+from packaging.version import parse as parse_version
 
 import ckan
 import ckan.model as model
@@ -210,8 +210,6 @@ def _init(self):
 
         self.ckan_version = ckan.__version__
         version = parse_version(self.ckan_version)
-        if not isinstance(version, Version):
-            raise ValueError(self.ckan_version)
 
         self.ckan_base_version = version.base_version
         if not version.is_prerelease:

ckan/lib/authenticator.py

@@ -4,6 +4,8 @@
 import ckan.plugins as plugins
 from typing import Any, Mapping, Optional
 
+from ckan.common import g, request
+from ckan.lib import captcha
 from ckan.model import User
 from . import signals
 
@@ -26,7 +28,21 @@ def default_authenticate(identity: 'Mapping[str, Any]') -> Optional["User"]:
     elif not user_obj.validate_password(identity['password']):
         log.debug('Login as %r failed - password not valid', login)
     else:
-        return user_obj
+        check_captcha = identity.get('check_captcha', True)
+        if check_captcha and g.recaptcha_publickey:
+            # Check for a valid reCAPTCHA response
+            try:
+                client_ip_address = request.remote_addr or 'Unknown IP Address'
+                captcha.check_recaptcha_v2_base(
+                    client_ip_address,
+                    request.form.get(u'g-recaptcha-response', '')
+                )
+                return user_obj
+            except captcha.CaptchaError:
+                log.warning('Login as %r failed - failed reCAPTCHA', login)
+                request.environ[u'captchaFailed'] = True
+        else:
+            return user_obj
     signals.failed_login.send(login)
     return None
 

ckan/lib/captcha.py

@@ -7,7 +7,7 @@
 
 
 def check_recaptcha(request: Request) -> None:
-    '''Check a user\'s recaptcha submission is valid, and raise CaptchaError
+    '''Check a user's recaptcha submission is valid, and raise CaptchaError
     on failure.'''
     recaptcha_private_key = config.get('ckan.recaptcha.privatekey')
     if not recaptcha_private_key:
@@ -17,8 +17,20 @@ def check_recaptcha(request: Request) -> None:
     client_ip_address = request.environ.get(
         'REMOTE_ADDR', 'Unknown IP Address')
 
-    # reCAPTCHA v2
     recaptcha_response_field = request.form.get('g-recaptcha-response', '')
+    check_recaptcha_v2_base(client_ip_address, recaptcha_response_field)
+
+
+def check_recaptcha_v2_base(client_ip_address: str,
+                            recaptcha_response_field: str) -> None:
+    '''Check a user's recaptcha submission is valid, and raise CaptchaError
+    on failure using discreet data'''
+    recaptcha_private_key = config.get('ckan.recaptcha.privatekey', '')
+    if not recaptcha_private_key:
+        # Recaptcha not enabled
+        return
+
+    # reCAPTCHA v2
     recaptcha_server_name = 'https://www.google.com/recaptcha/api/siteverify'
 
     # recaptcha_response_field will be unicode if there are foreign chars in

ckan/lib/jobs.py

@@ -43,7 +43,7 @@
 _queues: dict[str, rq.Queue] = {}
 
 
-def _connect() -> Redis:  # type: ignore
+def _connect() -> Redis:
     u'''
     Connect to Redis and tell RQ about it.
 

ckan/lib/redis.py

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-u'''
+'''
 Redis utilities.
 
 .. versionadded:: 2.7
@@ -21,8 +21,8 @@
 _connection_pool = None
 
 
-def connect_to_redis() -> Redis:  # type: ignore
-    u'''
+def connect_to_redis() -> Redis:
+    '''
     (Lazily) connect to Redis.
 
     The connection is set up but not actually established. The latter
@@ -42,7 +42,7 @@ def connect_to_redis() -> Redis:  # type: ignore
 
 
 def is_redis_available() -> bool:
-    u'''
+    '''
     Check whether Redis is available.
 
     :returns: The availability of Redis.
@@ -52,7 +52,7 @@ def is_redis_available() -> bool:
     '''
     redis_conn = connect_to_redis()
     try:
-        return redis_conn.ping()
+        return redis_conn.ping() is True
     except Exception:
         log.exception(u'Redis is not available')
         return False

ckan/lib/search/__init__.py

@@ -22,6 +22,7 @@
 
 from ckan.lib.search.common import (
     make_connection, SearchIndexError, SearchQueryError,  # type: ignore
+    SolrConnectionError, # type: ignore
     SearchError, is_available, SolrSettings, config
 )
 from ckan.lib.search.index import (

ckan/lib/search/common.py

@@ -4,6 +4,7 @@
 import datetime
 import logging
 import re
+from urllib3.exceptions import NewConnectionError
 from typing import Any, Optional
 
 import pysolr
@@ -29,6 +30,10 @@ class SearchQueryError(SearchError):
     pass
 
 
+class SolrConnectionError(NewConnectionError):
+    pass
+
+
 DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr/ckan'
 
 

ckan/lib/search/query.py

@@ -18,7 +18,7 @@
 
 from ckan.common import config
 from ckan.lib.search.common import (
-    make_connection, SearchError, SearchQueryError
+    make_connection, SearchError, SearchQueryError, SolrConnectionError
 )
 from ckan.types import Context
 
@@ -103,7 +103,7 @@ def _parse_local_params(local_params: str) -> list[Union[str, list[str]]]:
     {!type=dismax qf=myfield v='some value'} -> [['type', 'dismax'], ['qf', 'myfield'], ['v', 'some value']]
 
     """
-    key = Word(alphanums + "_")
+    key = Word(alphanums + "_.")
     value = QuotedString('"') | QuotedString("'") | Word(alphanums + "_$")
     pair = Group(key + Suppress("=") + value)
     expression = Suppress("{!") + OneOrMore(pair | key) + Suppress("}")
@@ -468,6 +468,10 @@ def _check_query_parser(param: str, value: Any):
                         "Can't determine Sort Order" in e.args[0] or \
                         'Unknown sort order' in e.args[0]:
                     raise SearchQueryError('Invalid "sort" parameter')
+
+                if "Failed to connect to server" in e.args[0]:
+                    raise SolrConnectionError("Connection Error", message=e.args[0])
+
             raise SearchError('SOLR returned an error running query: %r Error: %r' %
                               (query, e))
         self.count = solr_response.hits

ckan/logic/action/get.py

@@ -1004,7 +1004,7 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS
     context['package'] = pkg
     _check_access('package_show', context, data_dict)
 
-    if data_dict.get('use_default_schema', False):
+    if asbool(data_dict.get('use_default_schema', False)):
         context['schema'] = ckan.logic.schema.default_show_package_schema()
 
     package_dict = None

ckan/logic/action/patch.py

@@ -24,12 +24,10 @@ def package_patch(
     parameters unchanged, whereas the update methods deletes all parameters
     not explicitly provided in the data_dict.
 
-    You are able to partially update and/or create resources with
-    package_patch. If you are updating existing resources be sure to provide
-    the resource id. Existing resources excluded from the package_patch
-    data_dict will be removed. Resources in the package data_dict without
-    an id will be treated as new resources and will be added. New resources
-    added with the patch method do not create the default views.
+    To partially update resources or other metadata not at the top level
+    of a package use
+    :py:func:`~ckan.logic.action.update.package_revise` instead to maintain
+    existing nested values.
 
     You must be authorized to edit the dataset and the groups that it belongs
     to.

ckan/logic/action/update.py

@@ -774,7 +774,7 @@ def user_update(context: Context, data_dict: DataDict) -> ActionResult.UserUpdat
     '''Update a user account.
 
     Normal users can only update their own user accounts. Sysadmins can update
-    any user account. Can not modify exisiting user's name.
+    any user account and modify existing usernames.
 
     .. note:: Update methods may delete parameters not explicitly provided in the
         data_dict. If you want to edit only a specific attribute use `user_patch`

ckan/logic/validators.py

@@ -14,15 +14,14 @@
 from sqlalchemy.orm.exc import NoResultFound
 
 import ckan.lib.navl.dictization_functions as df
-import ckan.logic as logic
+from ckan import authz, logic
 import ckan.logic.converters as converters
 import ckan.lib.helpers as h
 from ckan.model import (MAX_TAG_LENGTH, MIN_TAG_LENGTH,
                         PACKAGE_NAME_MIN_LENGTH, PACKAGE_NAME_MAX_LENGTH,
                         PACKAGE_VERSION_MAX_LENGTH,
                         VOCABULARY_NAME_MAX_LENGTH,
                         VOCABULARY_NAME_MIN_LENGTH)
-import ckan.authz as authz
 from ckan.model.core import State
 
 from ckan.common import _
@@ -615,10 +614,13 @@ def user_name_validator(key: FlattenKey, data: FlattenDataDict,
             return
         else:
             # Otherwise return an error: there's already another user with that
-            # name, so you can create a new user with that name or update an
+            # name, so you can't create a new user with that name or update an
             # existing user's name to that name.
             errors[key].append(_('That login name is not available.'))
     elif user_obj_from_context:
+        requester = context.get('auth_user_obj', None)
+        if requester and authz.is_sysadmin(requester.name):
+            return
         old_user = model.User.get(user_obj_from_context.id)
         if old_user is not None and old_user.state != model.State.PENDING:
             errors[key].append(_('That login name can not be modified.'))