Support for Hermitian observables when used with qml.shadow_expval()

[poojithumeshrao]

Expected behavior

When the expectation value of a Hermitian observable is measured using qml.shadow_expval(), a float value has to be returned. Ideally, it must return a value close to qml.expval().

Actual behavior

"'NoneType' object is not iterable" error is thrown instead.

Additional information

No response

Source code

import pennylane as qml
import pennylane.numpy as np
from matplotlib import pyplot as plt
from pennylane import classical_shadow, shadow_expval, ClassicalShadow, expval

np.random.seed(666)

def pqc(f1,wires=range(2)):
    qml.ArbitraryStatePreparation(f1,wires=wires)

q = np.random.random((6))

mat = np.matrix(qml.matrix(pqc)(q))
mat = mat + mat.getH()
mat /= 2
obs = qml.Hermitian(mat,wires=[0,1])

H = obs

dev = qml.device("default.qubit", wires=range(2), shots=10000)
@qml.qnode(dev, interface="autograd")
def qnode(x, H):
    qml.Hadamard(0)
    qml.CNOT((0,1))
    qml.RX(x, wires=0)
    return shadow_expval(H)

x = np.array(0.5, requires_grad=True)

print(qnode(x, H), qml.grad(qnode)(x, H))

Tracebacks

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[8], line 32
     28     return shadow_expval(H)
     30 x = np.array(0.5, requires_grad=True)
---> 32 print(qnode(x, H), qml.grad(qnode)(x, H))

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/qnode.py:989, in QNode.__call__(self, *args, **kwargs)
    986     self.execute_kwargs.pop("mode")
    988 # pylint: disable=unexpected-keyword-arg
--> 989 res = qml.execute(
    990     (self._tape,),
    991     device=self.device,
    992     gradient_fn=self.gradient_fn,
    993     interface=self.interface,
    994     transform_program=self.transform_program,
    995     gradient_kwargs=self.gradient_kwargs,
    996     override_shots=override_shots,
    997     **self.execute_kwargs,
    998 )
   1000 res = res[0]
   1002 # convert result to the interface in case the qfunc has no parameters

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/interfaces/execution.py:757, in execute(tapes, device, gradient_fn, interface, transform_program, grad_on_execution, gradient_kwargs, cache, cachesize, max_diff, override_shots, expand_fn, max_expansion, device_batch_transform)
    754     raise ValueError("Gradient transforms cannot be used with grad_on_execution=True")
    756 ml_boundary_execute = _get_ml_boundary_execute(interface, _grad_on_execution)
--> 757 results = ml_boundary_execute(
    758     tapes, device, execute_fn, gradient_fn, gradient_kwargs, _n=1, max_diff=max_diff
    759 )
    761 results = batch_fn(results)
    762 return program_post_processing(results)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/interfaces/autograd.py:317, in execute(tapes, device, execute_fn, gradient_fn, gradient_kwargs, _n, max_diff)
    312 # pylint misidentifies autograd.builtins as a dict
    313 # pylint: disable=no-member
    314 parameters = autograd.builtins.tuple(
    315     [autograd.builtins.list(t.get_parameters()) for t in tapes]
    316 )
--> 317 return _execute(
    318     parameters,
    319     tapes=tapes,
    320     device=device,
    321     execute_fn=execute_fn,
    322     gradient_fn=gradient_fn,
    323     gradient_kwargs=gradient_kwargs,
    324     _n=_n,
    325     max_diff=max_diff,
    326 )[0]

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/autograd/tracer.py:48, in primitive.<locals>.f_wrapped(*args, **kwargs)
     46     return new_box(ans, trace, node)
     47 else:
---> 48     return f_raw(*args, **kwargs)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/interfaces/autograd.py:378, in _execute(parameters, tapes, device, execute_fn, gradient_fn, gradient_kwargs, _n, max_diff)
    360 if logger.isEnabledFor(logging.DEBUG):
    361     logger.debug(
    362         "Entry with args=(parameters=%s, tapes=%s, device=%s, execute_fn=%s, gradient_fn=%s, gradient_kwargs=%s, _n=%s, max_diff=%s) called by=%s",
    363         parameters,
   (...)
    375         "::L".join(str(i) for i in inspect.getouterframes(inspect.currentframe(), 2)[1][1:3]),
    376     )
--> 378 res, jacs = execute_fn(tapes, **gradient_kwargs)
    380 return res, jacs

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/interfaces/execution.py:623, in execute.<locals>.inner_execute_with_empty_jac(tapes, **_)
    622 def inner_execute_with_empty_jac(tapes, **_):
--> 623     return (inner_execute(tapes), [])

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/interfaces/execution.py:255, in _make_inner_execute.<locals>.inner_execute(tapes, **_)
    253 if numpy_only:
    254     tapes = tuple(qml.transforms.convert_to_numpy_parameters(t) for t in tapes)
--> 255 return cached_device_execution(tapes)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/interfaces/execution.py:377, in cache_execute.<locals>.wrapper(tapes, **kwargs)
    372         return (res, []) if return_tuple else res
    374 else:
    375     # execute all unique tapes that do not exist in the cache
    376     # convert to list as new device interface returns a tuple
--> 377     res = list(fn(tuple(execution_tapes.values()), **kwargs))
    379 final_res = []
    381 for i, tape in enumerate(tapes):

File ~/miniconda3/envs/pennylane/lib/python3.9/contextlib.py:79, in ContextDecorator.__call__.<locals>.inner(*args, **kwds)
     76 @wraps(func)
     77 def inner(*args, **kwds):
     78     with self._recreate_cm():
---> 79         return func(*args, **kwds)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/_qubit_device.py:629, in QubitDevice.batch_execute(self, circuits)
    624 for circuit in circuits:
    625     # we need to reset the device here, else it will
    626     # not start the next computation in the zero state
    627     self.reset()
--> 629     res = self.execute(circuit)
    630     results.append(res)
    632 if self.tracker.active:

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/_qubit_device.py:348, in QubitDevice.execute(self, circuit, **kwargs)
    345     results = self.shot_vec_statistics(circuit)
    347 else:
--> 348     results = self.statistics(circuit)
    349     single_measurement = len(circuit.measurements) == 1
    351     results = results[0] if single_measurement else tuple(results)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/_qubit_device.py:1089, in QubitDevice.statistics(self, circuit, shot_range, bin_size)
   1084     if len(measurements) > 1:
   1085         raise qml.QuantumFunctionError(
   1086             "Classical shadows cannot be returned in combination "
   1087             "with other return types"
   1088         )
-> 1089     result = self.shadow_expval(obs, circuit=circuit)
   1091 elif isinstance(m, MeasurementTransform):
   1092     result = m.process(tape=circuit, device=self)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/_qubit_device.py:1454, in QubitDevice.shadow_expval(self, obs, circuit)
   1452 bits, recipes = self.classical_shadow(obs, circuit)
   1453 shadow = qml.shadows.ClassicalShadow(bits, recipes, wire_map=obs.wires.tolist())
-> 1454 return shadow.expval(obs.H, obs.k)

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/shadows/classical_shadow.py:314, in ClassicalShadow.expval(self, H, k)
    310     H = [H]
    312 coeffs_and_words = [self._convert_to_pauli_words(h) for h in H]
    313 expvals = pauli_expval(
--> 314     self.bits, self.recipes, np.array([word for cw in coeffs_and_words for _, word in cw])
    315 )
    316 expvals = median_of_means(expvals, k, axis=0)
    317 expvals = expvals * np.array([coeff for cw in coeffs_and_words for coeff, _ in cw])

File ~/miniconda3/envs/pennylane/lib/python3.9/site-packages/pennylane/shadows/classical_shadow.py:314, in <listcomp>(.0)
    310     H = [H]
    312 coeffs_and_words = [self._convert_to_pauli_words(h) for h in H]
    313 expvals = pauli_expval(
--> 314     self.bits, self.recipes, np.array([word for cw in coeffs_and_words for _, word in cw])
    315 )
    316 expvals = median_of_means(expvals, k, axis=0)
    317 expvals = expvals * np.array([coeff for cw in coeffs_and_words for coeff, _ in cw])

TypeError: 'NoneType' object is not iterable

System information

Name: PennyLane
Version: 0.32.0
Summary: PennyLane is a Python quantum machine learning library by Xanadu Inc.
Home-page: https://github.com/PennyLaneAI/pennylane
Author: 
Author-email: 
License: Apache License 2.0
Location: /home/poojith/miniconda3/envs/pennylane/lib/python3.9/site-packages
Requires: appdirs, autograd, autoray, cachetools, networkx, numpy, pennylane-lightning, requests, rustworkx, scipy, semantic-version, toml, typing-extensions
Required-by: PennyLane-Lightning

Platform info:           Linux-4.19.0-23-amd64-x86_64-with-glibc2.28
Python version:          3.9.17
Numpy version:           1.23.5
Scipy version:           1.11.2
Installed devices:
- default.gaussian (PennyLane-0.32.0)
- default.mixed (PennyLane-0.32.0)
- default.qubit (PennyLane-0.32.0)
- default.qubit.autograd (PennyLane-0.32.0)
- default.qubit.jax (PennyLane-0.32.0)
- default.qubit.tf (PennyLane-0.32.0)
- default.qubit.torch (PennyLane-0.32.0)
- default.qutrit (PennyLane-0.32.0)
- null.qubit (PennyLane-0.32.0)
- lightning.qubit (PennyLane-Lightning-0.32.0)

Existing GitHub issues

• I have searched existing GitHub issues to make sure the issue does not already exist.

[CatalinaAlbornoz]

Hi @poojithumeshrao, thank you for opening this bug report!
Our team took a look at your bug and found that this seems to be expected behaviour but with an error message that doesn't say much.

A ClassicalShadow is a classical description of a quantum state that is capable of reproducing expectation values of local Pauli observables. In this case your error arises because the expected observable(s) aren't valid Pauli words. The Hermitian observable has no (obvious and a priori available) Pauli decomposition/representation.

Unfortunately it doesn't look like an easy fix from PennyLane's side but we will indeed improve the error message to make it easier to understand what's going on.

On the other hand we noticed that you're using PennyLane v0.32. If possible my recommendation would be to upgrade to the latest version. We will make a new release in two weeks so you can update your version now and again in two weeks to smooth out any possible issues.

If you want to learn more about classical shadows feel free to take a look at this demo.

[poojithumeshrao]

Hi, @CatalinaAlbornoz, thank you for the timely response.

After going through the traceback, I was able to make it work (at least in my case) by manually decomposing the Hermitian matrix using qml.pauli_decompose(). The modified code is as follows:

import pennylane as qml
import pennylane.numpy as np
from matplotlib import pyplot as plt
from pennylane import classical_shadow, shadow_expval, ClassicalShadow, expval

np.random.seed(666)

def pqc(f1,wires=range(2)):
    qml.ArbitraryStatePreparation(f1,wires=wires)

q = np.random.random((6))

mat = np.matrix(qml.matrix(pqc)(q))
mat = mat + mat.getH()
mat /= 2

H = qml.pauli_decompose(mat)

dev = qml.device("default.qubit", wires=range(2), shots=10000)
@qml.qnode(dev, interface="autograd")
def qnode(x, H):
    qml.Hadamard(0)
    qml.CNOT((0,1))
    qml.RX(x, wires=0)
    return shadow_expval([H])

x = np.array(0.5, requires_grad=True)

print(qnode(x, H), qml.grad(qnode)(x, H))

I am not sure if it works generally, but I thought it would also be useful for you.

[CatalinaAlbornoz]

Ah this is great! Thank you for sharing it @poojithumeshrao .

[trbromley]

Hi @poojithumeshrao! Thanks for posting this. I tweaked the title of this issue to reflect that we still have a valid feature request to eventually take care of, i.e., support for qml.Hermitian in classical shadows workflows.