Fix global SEND_DEFAULTS merging

Replace generic `combine` with
specific `merge_dicts_deep`,
`merge_dicts_shallow`,
`merge_dicts_one_level` or
`concat_lists`, depending on
appropriate behavior for each
message attribute.

Fixes merging global `SEND_DEFAULTS`
with message `esp_extra` for ESP APIs
that use nested payload structures.
And clarifies intent for other properties.

anymail/backends/base.py

@@ -18,13 +18,16 @@ from ..signals import post_send, pre_send
 from ..utils import (
     UNSET,
     Attachment,
-    combine,
+    concat_lists,
     force_non_lazy,
     force_non_lazy_dict,
     force_non_lazy_list,
     get_anymail_setting,
     is_lazy,
     last,
+    merge_dicts_deep,
+    merge_dicts_one_level,
+    merge_dicts_shallow,
     parse_address_list,
     parse_single_address,
 )
@@ -253,8 +256,7 @@ class BasePayload:
     #   attr: the property name
     #   combiner: optional function(default_value, value) -> value
     #     to combine settings defaults with the EmailMessage property value
-    #     (usually `combine` to merge, or `last` for message value to override default;
-    #     use `None` if settings defaults aren't supported)
+    #     (use `None` if settings defaults aren't supported)
     #   converter: optional function(value) -> value transformation
     #     (can be a callable or the string name of a Payload method, or `None`)
     #     The converter must force any Django lazy translation strings to text.
@@ -263,29 +265,29 @@ class BasePayload:
     base_message_attrs = (
         # Standard EmailMessage/EmailMultiAlternatives props
         ("from_email", last, parse_address_list),  # multiple from_emails are allowed
-        ("to", combine, parse_address_list),
-        ("cc", combine, parse_address_list),
-        ("bcc", combine, parse_address_list),
+        ("to", concat_lists, parse_address_list),
+        ("cc", concat_lists, parse_address_list),
+        ("bcc", concat_lists, parse_address_list),
         ("subject", last, force_non_lazy),
-        ("reply_to", combine, parse_address_list),
-        ("extra_headers", combine, force_non_lazy_dict),
+        ("reply_to", concat_lists, parse_address_list),
+        ("extra_headers", merge_dicts_shallow, force_non_lazy_dict),
         ("body", last, force_non_lazy),  # set_body handles content_subtype
-        ("alternatives", combine, "prepped_alternatives"),
-        ("attachments", combine, "prepped_attachments"),
+        ("alternatives", concat_lists, "prepped_alternatives"),
+        ("attachments", concat_lists, "prepped_attachments"),
     )
     anymail_message_attrs = (
         # Anymail expando-props
         ("envelope_sender", last, parse_single_address),
-        ("metadata", combine, force_non_lazy_dict),
+        ("metadata", merge_dicts_shallow, force_non_lazy_dict),
         ("send_at", last, "aware_datetime"),
-        ("tags", combine, force_non_lazy_list),
+        ("tags", concat_lists, force_non_lazy_list),
         ("track_clicks", last, None),
         ("track_opens", last, None),
         ("template_id", last, force_non_lazy),
-        ("merge_data", combine, force_non_lazy_dict),
-        ("merge_global_data", combine, force_non_lazy_dict),
-        ("merge_metadata", combine, force_non_lazy_dict),
-        ("esp_extra", combine, force_non_lazy_dict),
+        ("merge_data", merge_dicts_one_level, force_non_lazy_dict),
+        ("merge_global_data", merge_dicts_shallow, force_non_lazy_dict),
+        ("merge_metadata", merge_dicts_one_level, force_non_lazy_dict),
+        ("esp_extra", merge_dicts_deep, force_non_lazy_dict),
     )
     esp_message_attrs = ()  # subclasses can override
 

anymail/utils.py

@@ -2,6 +2,7 @@ import base64
 import mimetypes
 from base64 import b64encode
 from collections.abc import Mapping, MutableMapping
+from copy import copy, deepcopy
 from email.mime.base import MIMEBase
 from email.utils import formatdate, getaddresses, parsedate_to_datetime, unquote
 from urllib.parse import urlsplit, urlunsplit
@@ -20,17 +21,16 @@ BASIC_NUMERIC_TYPES = (int, float)
 UNSET = type("UNSET", (object,), {})  # Used as non-None default value
 
 
-def combine(*args):
+def concat_lists(*args):
     """
-    Combines all non-UNSET args, by shallow merging mappings and concatenating sequences
+    Combines all non-UNSET args, by concatenating lists (or sequence-like types).
+    Does not modify any args.
 
-    >>> combine({'a': 1, 'b': 2}, UNSET, {'b': 3, 'c': 4}, UNSET)
-    {'a': 1, 'b': 3, 'c': 4}
-    >>> combine([1, 2], UNSET, [3, 4], UNSET)
+    >>> concat_lists([1, 2], UNSET, [3, 4], UNSET)
     [1, 2, 3, 4]
-    >>> combine({'a': 1}, None, {'b': 2})  # None suppresses earlier args
-    {'b': 2}
-    >>> combine()
+    >>> concat_lists([1, 2], None, [3, 4])  # None suppresses earlier args
+    [3, 4]
+    >>> concat_lists()
     UNSET
 
     """
@@ -41,23 +41,93 @@ def combine(*args):
             result = UNSET
         elif value is not UNSET:
             if result is UNSET:
-                try:
-                    result = value.copy()  # will shallow merge if dict-like
-                except AttributeError:
-                    result = value  # will concatenate if sequence-like
+                result = list(value)
             else:
-                try:
-                    result.update(value)  # shallow merge if dict-like
-                except AttributeError:
-                    result = result + value  # concatenate if sequence-like
+                result = result + list(value)  # concatenate sequence-like
+    return result
+
+
+def merge_dicts_shallow(*args):
+    """
+    Shallow-merges all non-UNSET args.
+    Does not modify any args.
+
+    >>> merge_dicts_shallow({'a': 1, 'b': 2}, UNSET, {'b': 3, 'c': 4}, UNSET)
+    {'a': 1, 'b': 3, 'c': 4}
+    >>> merge_dicts_shallow({'a': {'a1': 1, 'a2': 2}}, {'a': {'a1': 11, 'a3': 33}})
+    {'a': {'a1': 11, 'a3': 33}}
+    >>> merge_dicts_shallow({'a': 1}, None, {'b': 2})  # None suppresses earlier args
+    {'b': 2}
+    >>> merge_dicts_shallow()
+    UNSET
+
+    """
+    result = UNSET
+    for value in args:
+        if value is None:
+            # None is a request to suppress any earlier values
+            result = UNSET
+        elif value is not UNSET:
+            if result is UNSET:
+                result = copy(value)
+            else:
+                result.update(value)
+    return result
+
+
+def merge_dicts_deep(*args):
+    """
+    Deep-merges all non-UNSET args.
+    Does not modify any args.
+
+    >>> merge_dicts_deep({'a': 1, 'b': 2}, UNSET, {'b': 3, 'c': 4}, UNSET)
+    {'a': 1, 'b': 3, 'c': 4}
+    >>> merge_dicts_deep({'a': {'a1': 1, 'a2': 2}}, {'a': {'a1': 11, 'a3': 33}})
+    {'a': {'a1': 11, 'a2': 2, 'a3': 33}}
+    >>> merge_dicts_deep({'a': 1}, None, {'b': 2})  # None suppresses earlier args
+    {'b': 2}
+    >>> merge_dicts_deep()
+    UNSET
+
+    """
+    result = UNSET
+    for value in args:
+        if value is None:
+            # None is a request to suppress any earlier values
+            result = UNSET
+        elif value is not UNSET:
+            if result is UNSET:
+                result = deepcopy(value)
+            else:
+                update_deep(result, value)
+    return result
+
+
+def merge_dicts_one_level(*args):
+    """
+    Mixture of merge_dicts_deep and merge_dicts_shallow:
+    Deep merges first level, shallow merges second level.
+    Does not modify any args.
+
+    (Useful for {"email": {options...}, ...} style dicts,
+    like merge_data: shallow merges the options for each email.)
+    """
+    result = UNSET
+    for value in args:
+        if value is None:
+            # None is a request to suppress any earlier values
+            result = UNSET
+        elif value is not UNSET:
+            if result is UNSET:
+                result = {}
+            for k, v in value.items():
+                result.setdefault(k, {}).update(v)
     return result
 
 
 def last(*args):
     """Returns the last of its args which is not UNSET.
 
-    (Essentially `combine` without the merge behavior)
-
     >>> last(1, 2, UNSET, 3, UNSET, UNSET)
     3
     >>> last(1, 2, None, UNSET)  # None suppresses earlier args

anymail/webhooks/mailgun.py

@@ -21,8 +21,8 @@ from ..signals import (
 )
 from ..utils import (
     UNSET,
-    combine,
     get_anymail_setting,
+    merge_dicts_shallow,
     parse_single_address,
     querydict_getfirst,
 )
@@ -341,7 +341,9 @@ class MailgunTrackingWebhookView(MailgunBaseWebhookView):
             if len(variables) >= 1:
                 # Each X-Mailgun-Variables value is JSON. Parse and merge them all into
                 # single dict:
-                metadata = combine(*[json.loads(value) for value in variables])
+                metadata = merge_dicts_shallow(
+                    *[json.loads(value) for value in variables]
+                )
 
         elif event_type in self._known_legacy_event_fields:
             # For other events, we must extract from the POST fields, ignoring known