mirror of
https://github.com/mealie-recipes/mealie.git
synced 2026-07-25 00:50:14 -04:00
fix: harden server-initiated HTTP against SSRF and DNS rebinding
Route all server-initiated outbound HTTP through one SSRF-protected layer: - Resolve each host once, validate every resolved address (private, loopback, link-local, CGNAT 100.64/10, multicast, reserved, unspecified, IPv4-mapped IPv6), and reject if any is unsafe. - Pin the connection to the validated address via curl's RESOLVE option so curl cannot re-resolve to a rebound IP (closes the validate-then-connect TOCTOU). Uses getaddrinfo so IPv6 records are checked too. - Add a sync SafeTransport and safehttp.post, and move the webhook and recipe-action paths off raw requests.post so they are guarded like the scraper. - Add HTTP_ALLOW_LIST / HTTP_DISALLOW_LIST settings (deny > allow > default) so operators can reach an internal server or block extra hosts. Apprise notifications egress through the third-party library and are not covered here.
This commit is contained in:
@@ -30,6 +30,8 @@
|
|||||||
| SECURITY_MAX_LOGIN_ATTEMPTS | 5 | Maximum times a user can provide an invalid password before their account is locked |
|
| SECURITY_MAX_LOGIN_ATTEMPTS | 5 | Maximum times a user can provide an invalid password before their account is locked |
|
||||||
| SECURITY_USER_LOCKOUT_TIME | 24 | Time in hours for how long a users account is locked |
|
| SECURITY_USER_LOCKOUT_TIME | 24 | Time in hours for how long a users account is locked |
|
||||||
| ALLOWED_IFRAME_HOSTS | `""` | Comma-separated extra hostnames allowed as `<iframe>` sources in recipe content. Extends the built-in list of trusted video providers (YouTube, Vimeo). Subdomains are included automatically. Only `https` sources are permitted. Adding hosts here opts into rendering embeds from those origins to all viewers, including the public, so add only origins you trust. |
|
| ALLOWED_IFRAME_HOSTS | `""` | Comma-separated extra hostnames allowed as `<iframe>` sources in recipe content. Extends the built-in list of trusted video providers (YouTube, Vimeo). Subdomains are included automatically. Only `https` sources are permitted. Adding hosts here opts into rendering embeds from those origins to all viewers, including the public, so add only origins you trust. |
|
||||||
|
| HTTP_ALLOW_LIST | `""` | Comma-separated hosts or CIDRs that server-initiated requests (recipe scraping, webhooks, recipe actions) may reach even when they resolve to an otherwise-blocked private/internal address. Use to allow a known internal server. |
|
||||||
|
| HTTP_DISALLOW_LIST | `""` | Comma-separated hosts or CIDRs that server-initiated requests may never reach, even if public. Takes precedence over `HTTP_ALLOW_LIST`. |
|
||||||
|
|
||||||
### Database
|
### Database
|
||||||
|
|
||||||
|
|||||||
@@ -173,6 +173,23 @@ class AppSettings(AppLoggingSettings):
|
|||||||
extra = [host.strip().lower() for host in self.ALLOWED_IFRAME_HOSTS.split(",") if host.strip()]
|
extra = [host.strip().lower() for host in self.ALLOWED_IFRAME_HOSTS.split(",") if host.strip()]
|
||||||
return list(dict.fromkeys(DEFAULT_ALLOWED_IFRAME_HOSTS + extra))
|
return list(dict.fromkeys(DEFAULT_ALLOWED_IFRAME_HOSTS + extra))
|
||||||
|
|
||||||
|
HTTP_ALLOW_LIST: str = ""
|
||||||
|
"""Comma-separated hosts or CIDRs that server-initiated requests (recipe scraping, webhooks,
|
||||||
|
recipe actions) may reach even if they resolve to an otherwise-blocked private/internal address.
|
||||||
|
Use this to allow a known internal server."""
|
||||||
|
|
||||||
|
HTTP_DISALLOW_LIST: str = ""
|
||||||
|
"""Comma-separated hosts or CIDRs that server-initiated requests may never reach, even if public.
|
||||||
|
Takes precedence over `HTTP_ALLOW_LIST`."""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def http_allow_list(self) -> list[str]:
|
||||||
|
return [host.strip() for host in self.HTTP_ALLOW_LIST.split(",") if host.strip()]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def http_disallow_list(self) -> list[str]:
|
||||||
|
return [host.strip() for host in self.HTTP_DISALLOW_LIST.split(",") if host.strip()]
|
||||||
|
|
||||||
DAILY_SCHEDULE_TIME: str = "23:45"
|
DAILY_SCHEDULE_TIME: str = "23:45"
|
||||||
"""Local server time, in HH:MM format. See `DAILY_SCHEDULE_TIME_UTC` for the parsed UTC equivalent"""
|
"""Local server time, in HH:MM format. See `DAILY_SCHEDULE_TIME_UTC` for the parsed UTC equivalent"""
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,17 @@
|
|||||||
from .transport import AsyncSafeTransport, ForcedTimeoutException, InvalidDomainError
|
from .transport import (
|
||||||
|
AsyncSafeTransport,
|
||||||
|
ForcedTimeoutException,
|
||||||
|
InvalidDomainError,
|
||||||
|
SafeTransport,
|
||||||
|
is_blocked_ip,
|
||||||
|
post,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"AsyncSafeTransport",
|
"AsyncSafeTransport",
|
||||||
|
"SafeTransport",
|
||||||
"ForcedTimeoutException",
|
"ForcedTimeoutException",
|
||||||
"InvalidDomainError",
|
"InvalidDomainError",
|
||||||
|
"is_blocked_ip",
|
||||||
|
"post",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,9 +1,19 @@
|
|||||||
import ipaddress
|
import ipaddress
|
||||||
import logging
|
import logging
|
||||||
import socket
|
import socket
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from httpx_curl_cffi import AsyncCurlTransport
|
from curl_cffi import CurlOpt
|
||||||
|
from httpx_curl_cffi import AsyncCurlTransport, CurlTransport
|
||||||
|
|
||||||
|
IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
|
||||||
|
IPNetwork = ipaddress.IPv4Network | ipaddress.IPv6Network
|
||||||
|
|
||||||
|
# Carrier-grade NAT. `is_private` does not cover this range on every Python
|
||||||
|
# version, so we check it explicitly.
|
||||||
|
CGNAT_NETWORK = ipaddress.ip_network("100.64.0.0/10")
|
||||||
|
|
||||||
|
|
||||||
class ForcedTimeoutException(Exception):
|
class ForcedTimeoutException(Exception):
|
||||||
@@ -16,61 +26,179 @@ class ForcedTimeoutException(Exception):
|
|||||||
|
|
||||||
class InvalidDomainError(Exception):
|
class InvalidDomainError(Exception):
|
||||||
"""
|
"""
|
||||||
Raised when a request is made to a local IP address.
|
Raised when a request targets a disallowed (e.g. local/internal) address.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
class AsyncSafeTransport(AsyncCurlTransport):
|
def _normalize(ip: IPAddress) -> IPAddress:
|
||||||
|
# Unwrap IPv4-mapped IPv6 (e.g. ::ffff:127.0.0.1) so a mapped address can't
|
||||||
|
# dodge the IPv4 checks below.
|
||||||
|
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
||||||
|
return ip.ipv4_mapped
|
||||||
|
return ip
|
||||||
|
|
||||||
|
|
||||||
|
def is_blocked_ip(ip: IPAddress) -> bool:
|
||||||
|
"""Return True if connecting to this address should never be allowed by default."""
|
||||||
|
ip = _normalize(ip)
|
||||||
|
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
|
||||||
|
return True
|
||||||
|
return isinstance(ip, ipaddress.IPv4Address) and ip in CGNAT_NETWORK
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_host_list(entries: Sequence[str]) -> tuple[set[str], list[IPNetwork]]:
|
||||||
|
"""Split configured entries into exact hostnames and IP networks (CIDRs)."""
|
||||||
|
hosts: set[str] = set()
|
||||||
|
networks: list[IPNetwork] = []
|
||||||
|
for entry in entries:
|
||||||
|
cleaned = entry.strip().lower()
|
||||||
|
if not cleaned:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
networks.append(ipaddress.ip_network(cleaned, strict=False))
|
||||||
|
except ValueError:
|
||||||
|
hosts.add(cleaned)
|
||||||
|
return hosts, networks
|
||||||
|
|
||||||
|
|
||||||
|
def _matches(host: str, ips: Sequence[IPAddress], hosts: set[str], networks: list[IPNetwork]) -> bool:
|
||||||
|
if host.lower() in hosts:
|
||||||
|
return True
|
||||||
|
for ip in ips:
|
||||||
|
normalized = _normalize(ip)
|
||||||
|
for network in networks:
|
||||||
|
if normalized.version == network.version and normalized in network:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class _SafeTransportMixin:
|
||||||
"""
|
"""
|
||||||
A wrapper around the httpx-curl-cffi transport class that enforces a timeout value
|
Shared SSRF protection for the sync and async curl transports.
|
||||||
and that the request is not made to a local IP address.
|
|
||||||
|
On each request it resolves the host once, validates every resolved address,
|
||||||
|
and pins the connection to the validated address(es) via curl's RESOLVE option
|
||||||
|
so curl cannot re-resolve to a different (e.g. rebound) IP.
|
||||||
|
|
||||||
|
Not safe for concurrent requests on a single transport instance: the pin is
|
||||||
|
applied through the shared session. Mealie uses a fresh transport per scrape
|
||||||
|
and per outbound POST, and redirects are followed sequentially.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
timeout: int = 15
|
timeout: int = 15
|
||||||
|
# set by the curl transport base class this mixin is combined with
|
||||||
|
_session: Any
|
||||||
|
|
||||||
def __init__(self, log: logging.Logger | None = None, **kwargs):
|
def __init__(
|
||||||
self.timeout = kwargs.pop("timeout", self.timeout)
|
self,
|
||||||
|
*,
|
||||||
|
log: logging.Logger | None = None,
|
||||||
|
allow_hosts: Sequence[str] = (),
|
||||||
|
deny_hosts: Sequence[str] = (),
|
||||||
|
timeout: int | None = None,
|
||||||
|
**kwargs,
|
||||||
|
) -> None:
|
||||||
self._log = log
|
self._log = log
|
||||||
|
self._allow = _parse_host_list(allow_hosts)
|
||||||
|
self._deny = _parse_host_list(deny_hosts)
|
||||||
|
if timeout is not None:
|
||||||
|
self.timeout = timeout
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
def _validate(self, request: httpx.Request) -> list[str] | None:
|
||||||
# override timeout value for _all_ requests
|
"""Validate the request target. Returns the curl RESOLVE pins, or None for an IP literal."""
|
||||||
|
# Force our timeout onto every request.
|
||||||
request.extensions["timeout"] = httpx.Timeout(self.timeout, pool=self.timeout).as_dict()
|
request.extensions["timeout"] = httpx.Timeout(self.timeout, pool=self.timeout).as_dict()
|
||||||
|
|
||||||
# validate the request is not attempting to connect to a local IP
|
host = request.url.host
|
||||||
# This is a security measure to prevent SSRF attacks
|
port = request.url.port or (443 if request.url.scheme == "https" else 80)
|
||||||
|
|
||||||
ip: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None
|
|
||||||
|
|
||||||
netloc = request.url.netloc.decode()
|
|
||||||
if ":" in netloc: # Either an IP, or a hostname:port combo
|
|
||||||
netloc_parts = netloc.split(":")
|
|
||||||
|
|
||||||
netloc = netloc_parts[0]
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ip = ipaddress.ip_address(netloc)
|
literal: IPAddress | None = ipaddress.ip_address(host)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
if self._log:
|
literal = None
|
||||||
self._log.debug(f"failed to parse ip for {netloc=} falling back to domain resolution")
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Request is a domain or a hostname.
|
if literal is not None:
|
||||||
if not ip:
|
ips: list[IPAddress] = [literal]
|
||||||
if self._log:
|
else:
|
||||||
self._log.debug(f"resolving IP for domain: {netloc}")
|
try:
|
||||||
|
infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||||
|
except socket.gaierror as e:
|
||||||
|
raise InvalidDomainError(f"could not resolve host: {host}") from e
|
||||||
|
|
||||||
ip_str = socket.gethostbyname(netloc)
|
ips = []
|
||||||
ip = ipaddress.ip_address(ip_str)
|
seen: set[str] = set()
|
||||||
|
for info in infos:
|
||||||
|
addr = str(info[4][0]).split("%")[0] # drop any IPv6 scope id
|
||||||
|
if addr not in seen:
|
||||||
|
seen.add(addr)
|
||||||
|
ips.append(ipaddress.ip_address(addr))
|
||||||
|
|
||||||
if self._log:
|
deny_hosts, deny_networks = self._deny
|
||||||
self._log.debug(f"resolved IP for domain: {netloc} -> {ip}")
|
if _matches(host, ips, deny_hosts, deny_networks):
|
||||||
|
self._warn(request, "target is on the disallow list")
|
||||||
|
raise InvalidDomainError(f"request blocked by disallow list: {request.url}")
|
||||||
|
|
||||||
if ip.is_private:
|
allow_hosts, allow_networks = self._allow
|
||||||
if self._log:
|
if not _matches(host, ips, allow_hosts, allow_networks):
|
||||||
self._log.warning(f"invalid request on local resource: {request.url} -> {ip}")
|
for ip in ips:
|
||||||
|
if is_blocked_ip(ip):
|
||||||
|
self._warn(request, f"resolves to non-public address {ip}")
|
||||||
raise InvalidDomainError(f"invalid request on local resource: {request.url} -> {ip}")
|
raise InvalidDomainError(f"invalid request on local resource: {request.url} -> {ip}")
|
||||||
|
|
||||||
|
if literal is not None:
|
||||||
|
# curl connects straight to the literal address; nothing to pin.
|
||||||
|
return None
|
||||||
|
return [f"{host}:{port}:{ip}" for ip in ips]
|
||||||
|
|
||||||
|
def _warn(self, request: httpx.Request, reason: str) -> None:
|
||||||
|
if self._log:
|
||||||
|
self._log.warning(f"[safehttp] blocked request to {request.url}: {reason}")
|
||||||
|
|
||||||
|
def _apply_pin(self, resolve: list[str] | None) -> None:
|
||||||
|
# curl_cffi applies the session's curl_options to each request, so setting
|
||||||
|
# RESOLVE here pins this request to the address we just validated.
|
||||||
|
options = self._session.curl_options
|
||||||
|
if resolve:
|
||||||
|
options[CurlOpt.RESOLVE] = resolve
|
||||||
|
else:
|
||||||
|
options.pop(CurlOpt.RESOLVE, None)
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncSafeTransport(_SafeTransportMixin, AsyncCurlTransport):
|
||||||
|
"""Async curl transport that blocks SSRF and pins the connection to a validated IP."""
|
||||||
|
|
||||||
|
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||||
|
resolve = self._validate(request)
|
||||||
|
self._apply_pin(resolve)
|
||||||
return await super().handle_async_request(request)
|
return await super().handle_async_request(request)
|
||||||
|
|
||||||
|
|
||||||
|
class SafeTransport(_SafeTransportMixin, CurlTransport):
|
||||||
|
"""Sync counterpart of AsyncSafeTransport for server-initiated POSTs (webhooks, recipe actions)."""
|
||||||
|
|
||||||
|
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||||
|
resolve = self._validate(request)
|
||||||
|
self._apply_pin(resolve)
|
||||||
|
return super().handle_request(request)
|
||||||
|
|
||||||
|
|
||||||
|
def post(
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
json=None,
|
||||||
|
timeout: int = 15,
|
||||||
|
allow_hosts: Sequence[str] = (),
|
||||||
|
deny_hosts: Sequence[str] = (),
|
||||||
|
**kwargs,
|
||||||
|
) -> httpx.Response:
|
||||||
|
"""Perform a sync POST through the SSRF-protected transport.
|
||||||
|
|
||||||
|
Drop-in for `requests.post(url, json=..., timeout=...)` on server-initiated
|
||||||
|
requests to user-supplied URLs. Redirects are followed and re-validated per hop.
|
||||||
|
"""
|
||||||
|
transport = SafeTransport(allow_hosts=allow_hosts, deny_hosts=deny_hosts, timeout=timeout)
|
||||||
|
with httpx.Client(transport=transport, follow_redirects=True) as client:
|
||||||
|
return client.post(url, json=json, timeout=timeout, **kwargs)
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
|
|
||||||
import requests
|
|
||||||
from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, status
|
from fastapi import APIRouter, BackgroundTasks, Body, Depends, HTTPException, status
|
||||||
from fastapi.encoders import jsonable_encoder
|
from fastapi.encoders import jsonable_encoder
|
||||||
from pydantic import UUID4
|
from pydantic import UUID4
|
||||||
|
|
||||||
|
from mealie.core.config import get_app_settings
|
||||||
from mealie.core.exceptions import NoEntryFound
|
from mealie.core.exceptions import NoEntryFound
|
||||||
|
from mealie.pkgs import safehttp
|
||||||
from mealie.routes._base.base_controllers import BaseUserController
|
from mealie.routes._base.base_controllers import BaseUserController
|
||||||
from mealie.routes._base.controller import controller
|
from mealie.routes._base.controller import controller
|
||||||
from mealie.routes._base.mixins import HttpRepo
|
from mealie.routes._base.mixins import HttpRepo
|
||||||
@@ -79,7 +80,7 @@ class GroupRecipeActionController(BaseUserController):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if recipe_action.action_type == GroupRecipeActionType.post.value:
|
if recipe_action.action_type == GroupRecipeActionType.post.value:
|
||||||
task_action = requests.post
|
task_action = safehttp.post
|
||||||
else:
|
else:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status.HTTP_400_BAD_REQUEST,
|
status.HTTP_400_BAD_REQUEST,
|
||||||
@@ -95,10 +96,13 @@ class GroupRecipeActionController(BaseUserController):
|
|||||||
detail=ErrorResponse.respond(message="Not found."),
|
detail=ErrorResponse.respond(message="Not found."),
|
||||||
) from e
|
) from e
|
||||||
|
|
||||||
|
settings = get_app_settings()
|
||||||
payload = GroupRecipeActionPayload(action=recipe_action, content=recipe, recipe_scale=recipe_scale)
|
payload = GroupRecipeActionPayload(action=recipe_action, content=recipe, recipe_scale=recipe_scale)
|
||||||
bg_tasks.add_task(
|
bg_tasks.add_task(
|
||||||
task_action,
|
task_action,
|
||||||
url=recipe_action.url,
|
url=recipe_action.url,
|
||||||
json=jsonable_encoder(payload.model_dump()),
|
json=jsonable_encoder(payload.model_dump()),
|
||||||
timeout=15,
|
timeout=15,
|
||||||
|
allow_hosts=settings.http_allow_list,
|
||||||
|
deny_hosts=settings.http_disallow_list,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
from typing import Protocol
|
from typing import Protocol
|
||||||
|
|
||||||
import apprise
|
import apprise
|
||||||
import requests
|
|
||||||
from fastapi.encoders import jsonable_encoder
|
from fastapi.encoders import jsonable_encoder
|
||||||
|
|
||||||
|
from mealie.core.config import get_app_settings
|
||||||
|
from mealie.pkgs import safehttp
|
||||||
from mealie.services.event_bus_service.event_types import Event
|
from mealie.services.event_bus_service.event_types import Event
|
||||||
|
|
||||||
|
|
||||||
@@ -43,7 +44,14 @@ class WebhookPublisher:
|
|||||||
|
|
||||||
def publish(self, event: Event, notification_urls: list[str]):
|
def publish(self, event: Event, notification_urls: list[str]):
|
||||||
event_payload = jsonable_encoder(event)
|
event_payload = jsonable_encoder(event)
|
||||||
|
settings = get_app_settings()
|
||||||
for url in notification_urls:
|
for url in notification_urls:
|
||||||
r = requests.post(url, json=event_payload, timeout=15)
|
r = safehttp.post(
|
||||||
|
url,
|
||||||
|
json=event_payload,
|
||||||
|
timeout=15,
|
||||||
|
allow_hosts=settings.http_allow_list,
|
||||||
|
deny_hosts=settings.http_disallow_list,
|
||||||
|
)
|
||||||
if self.hard_fail:
|
if self.hard_fail:
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
|||||||
from httpx import AsyncClient, Response
|
from httpx import AsyncClient, Response
|
||||||
from pydantic import UUID4
|
from pydantic import UUID4
|
||||||
|
|
||||||
|
from mealie.core.config import get_app_settings
|
||||||
from mealie.pkgs import img, safehttp
|
from mealie.pkgs import img, safehttp
|
||||||
from mealie.schema.recipe.recipe import Recipe
|
from mealie.schema.recipe.recipe import Recipe
|
||||||
from mealie.schema.recipe.recipe_image_types import RecipeImageTypes
|
from mealie.schema.recipe.recipe_image_types import RecipeImageTypes
|
||||||
@@ -31,11 +32,20 @@ async def largest_content_len(urls: list[str]) -> tuple[str, int]:
|
|||||||
|
|
||||||
max_concurrency = 10
|
max_concurrency = 10
|
||||||
|
|
||||||
async def do(client: AsyncClient, url: str) -> Response:
|
settings = get_app_settings()
|
||||||
|
|
||||||
|
async def do(url: str) -> Response:
|
||||||
|
# A dedicated transport per request: the safe transport pins each request
|
||||||
|
# through its session, so it must not be shared across concurrent requests.
|
||||||
|
transport = safehttp.AsyncSafeTransport(
|
||||||
|
allow_hosts=settings.http_allow_list,
|
||||||
|
deny_hosts=settings.http_disallow_list,
|
||||||
|
impersonate="chrome",
|
||||||
|
)
|
||||||
|
async with AsyncClient(transport=transport) as client:
|
||||||
return await client.head(url)
|
return await client.head(url)
|
||||||
|
|
||||||
async with AsyncClient(transport=safehttp.AsyncSafeTransport(impersonate="chrome")) as client:
|
tasks = [do(url) for url in urls]
|
||||||
tasks = [do(client, url) for url in urls]
|
|
||||||
responses: list[Response] = await gather_with_concurrency(max_concurrency, *tasks, ignore_exceptions=True)
|
responses: list[Response] = await gather_with_concurrency(max_concurrency, *tasks, ignore_exceptions=True)
|
||||||
for response in responses:
|
for response in responses:
|
||||||
len_int = int(response.headers.get("Content-Length", 0))
|
len_int = int(response.headers.get("Content-Length", 0))
|
||||||
@@ -146,7 +156,13 @@ class RecipeDataService(BaseService):
|
|||||||
file_name = f"{self.recipe_id!s}.{ext}"
|
file_name = f"{self.recipe_id!s}.{ext}"
|
||||||
file_path = Recipe.directory_from_id(self.recipe_id).joinpath("images", file_name)
|
file_path = Recipe.directory_from_id(self.recipe_id).joinpath("images", file_name)
|
||||||
|
|
||||||
async with AsyncClient(transport=safehttp.AsyncSafeTransport(impersonate="chrome")) as client:
|
settings = get_app_settings()
|
||||||
|
transport = safehttp.AsyncSafeTransport(
|
||||||
|
allow_hosts=settings.http_allow_list,
|
||||||
|
deny_hosts=settings.http_disallow_list,
|
||||||
|
impersonate="chrome",
|
||||||
|
)
|
||||||
|
async with AsyncClient(transport=transport) as client:
|
||||||
try:
|
try:
|
||||||
r = await client.get(image_url_str)
|
r = await client.get(image_url_str)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from w3lib.html import get_base_url
|
|||||||
from yt_dlp.extractor.generic import GenericIE
|
from yt_dlp.extractor.generic import GenericIE
|
||||||
|
|
||||||
from mealie.core import exceptions
|
from mealie.core import exceptions
|
||||||
|
from mealie.core.config import get_app_settings
|
||||||
from mealie.core.dependencies.dependencies import get_temporary_path
|
from mealie.core.dependencies.dependencies import get_temporary_path
|
||||||
from mealie.core.root_logger import get_logger
|
from mealie.core.root_logger import get_logger
|
||||||
from mealie.lang.providers import Translator
|
from mealie.lang.providers import Translator
|
||||||
@@ -76,7 +77,10 @@ async def safe_scrape_html(url: str) -> str:
|
|||||||
html_bytes = b""
|
html_bytes = b""
|
||||||
response = None
|
response = None
|
||||||
|
|
||||||
|
settings = get_app_settings()
|
||||||
transport = safehttp.AsyncSafeTransport(
|
transport = safehttp.AsyncSafeTransport(
|
||||||
|
allow_hosts=settings.http_allow_list,
|
||||||
|
deny_hosts=settings.http_disallow_list,
|
||||||
impersonate=impersonation,
|
impersonate=impersonation,
|
||||||
default_headers=True,
|
default_headers=True,
|
||||||
verify=False, # disable SSL verification since we can handle untrusted data and some sites don't have certs
|
verify=False, # disable SSL verification since we can handle untrusted data and some sites don't have certs
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
from uuid import UUID, uuid4
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import requests
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from mealie.pkgs import safehttp
|
||||||
from mealie.schema.household.group_recipe_action import (
|
from mealie.schema.household.group_recipe_action import (
|
||||||
CreateGroupRecipeAction,
|
CreateGroupRecipeAction,
|
||||||
GroupRecipeActionOut,
|
GroupRecipeActionOut,
|
||||||
@@ -17,8 +17,8 @@ from tests.utils.fixture_schemas import TestUser
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def mock_requests_post(monkeypatch: pytest.MonkeyPatch):
|
def mock_outbound_post(monkeypatch: pytest.MonkeyPatch):
|
||||||
monkeypatch.setattr(requests, "post", lambda *args, **kwargs: None)
|
monkeypatch.setattr(safehttp, "post", lambda *args, **kwargs: None)
|
||||||
|
|
||||||
|
|
||||||
def create_action(action_type: GroupRecipeActionType = GroupRecipeActionType.link) -> CreateGroupRecipeAction:
|
def create_action(action_type: GroupRecipeActionType = GroupRecipeActionType.link) -> CreateGroupRecipeAction:
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ def test_delete_webhook(api_client: TestClient, webhook_data, unique_user: TestU
|
|||||||
def test_post_test_webhook(
|
def test_post_test_webhook(
|
||||||
monkeypatch: pytest.MonkeyPatch, api_client: TestClient, unique_user: TestUser, webhook_data
|
monkeypatch: pytest.MonkeyPatch, api_client: TestClient, unique_user: TestUser, webhook_data
|
||||||
):
|
):
|
||||||
# Mock the requests.post to avoid actual HTTP calls
|
# Mock the outbound post to avoid actual HTTP calls
|
||||||
class MockResponse:
|
class MockResponse:
|
||||||
status_code = 200
|
status_code = 200
|
||||||
|
|
||||||
@@ -101,7 +101,7 @@ def test_post_test_webhook(
|
|||||||
mock_calls.append((args, kwargs))
|
mock_calls.append((args, kwargs))
|
||||||
return MockResponse()
|
return MockResponse()
|
||||||
|
|
||||||
monkeypatch.setattr("mealie.services.event_bus_service.publisher.requests.post", mock_post)
|
monkeypatch.setattr("mealie.services.event_bus_service.publisher.safehttp.post", mock_post)
|
||||||
|
|
||||||
# Create a webhook and post it
|
# Create a webhook and post it
|
||||||
response = api_client.post(
|
response = api_client.post(
|
||||||
@@ -124,7 +124,7 @@ def test_post_test_webhook(
|
|||||||
test_message = "This is a test webhook message"
|
test_message = "This is a test webhook message"
|
||||||
post_test_webhook(webhook, test_message)
|
post_test_webhook(webhook, test_message)
|
||||||
|
|
||||||
# Verify that requests.post was called with the correct parameters
|
# Verify that the outbound post was called with the correct parameters
|
||||||
assert len(mock_calls) == 1
|
assert len(mock_calls) == 1
|
||||||
args, kwargs = mock_calls[0]
|
args, kwargs = mock_calls[0]
|
||||||
|
|
||||||
|
|||||||
189
tests/unit_tests/pkgs/test_safehttp.py
Normal file
189
tests/unit_tests/pkgs/test_safehttp.py
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
import http.server
|
||||||
|
import ipaddress
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from mealie.pkgs import safehttp
|
||||||
|
from mealie.pkgs.safehttp import transport as safehttp_transport
|
||||||
|
from mealie.pkgs.safehttp.transport import (
|
||||||
|
AsyncSafeTransport,
|
||||||
|
InvalidDomainError,
|
||||||
|
is_blocked_ip,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _request(url: str) -> httpx.Request:
|
||||||
|
return httpx.Request("GET", url)
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_resolver(monkeypatch, ips: list[str]) -> None:
|
||||||
|
def fake_getaddrinfo(host, port, *args, **kwargs):
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port)) for ip in ips]
|
||||||
|
|
||||||
|
monkeypatch.setattr(safehttp_transport.socket, "getaddrinfo", fake_getaddrinfo)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("ip", "blocked"),
|
||||||
|
[
|
||||||
|
("8.8.8.8", False),
|
||||||
|
("2606:4700:4700::1111", False),
|
||||||
|
("10.0.0.1", True),
|
||||||
|
("172.16.5.5", True),
|
||||||
|
("192.168.1.1", True),
|
||||||
|
("127.0.0.1", True),
|
||||||
|
("169.254.169.254", True), # cloud metadata / link-local
|
||||||
|
("100.64.0.1", True), # CGNAT, not covered by is_private on all versions
|
||||||
|
("0.0.0.0", True),
|
||||||
|
("224.0.0.1", True), # multicast
|
||||||
|
("::1", True),
|
||||||
|
("fe80::1", True),
|
||||||
|
("fc00::1", True),
|
||||||
|
("::ffff:127.0.0.1", True), # IPv4-mapped loopback
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_is_blocked_ip(ip: str, blocked: bool):
|
||||||
|
assert is_blocked_ip(ipaddress.ip_address(ip)) is blocked
|
||||||
|
|
||||||
|
|
||||||
|
def test_blocks_host_resolving_to_private(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["10.0.0.5"])
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("https://evil.example/"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_allows_public_host_and_returns_pin(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["93.184.216.34"])
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
resolve = transport._validate(_request("https://example.test/path"))
|
||||||
|
assert resolve == ["example.test:443:93.184.216.34"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_covers_all_resolved_addresses(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["93.184.216.34", "93.184.216.35"])
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
resolve = transport._validate(_request("http://example.test/"))
|
||||||
|
assert resolve == ["example.test:80:93.184.216.34", "example.test:80:93.184.216.35"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejects_when_any_resolved_address_is_unsafe(monkeypatch):
|
||||||
|
# one public, one private -> reject the whole request
|
||||||
|
_patch_resolver(monkeypatch, ["93.184.216.34", "127.0.0.1"])
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("https://mixed.example/"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_ip_literal_public_needs_no_pin():
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
assert transport._validate(_request("https://93.184.216.34/")) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_ip_literal_metadata_blocked():
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("http://169.254.169.254/latest/meta-data/"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_ipv6_resolution_is_validated(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["::1"])
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("https://v6.example/"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_list_bypasses_block(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["10.0.0.5"])
|
||||||
|
transport = AsyncSafeTransport(allow_hosts=["internal.example"])
|
||||||
|
resolve = transport._validate(_request("https://internal.example/"))
|
||||||
|
assert resolve == ["internal.example:443:10.0.0.5"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_list_cidr_bypasses_block(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["10.1.2.3"])
|
||||||
|
transport = AsyncSafeTransport(allow_hosts=["10.0.0.0/8"])
|
||||||
|
assert transport._validate(_request("https://internal.example/")) == ["internal.example:443:10.1.2.3"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_deny_list_overrides_allow(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["93.184.216.34"])
|
||||||
|
transport = AsyncSafeTransport(allow_hosts=["blocked.example"], deny_hosts=["blocked.example"])
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("https://blocked.example/"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_deny_list_blocks_public_host(monkeypatch):
|
||||||
|
_patch_resolver(monkeypatch, ["93.184.216.34"])
|
||||||
|
transport = AsyncSafeTransport(deny_hosts=["93.184.216.0/24"])
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("https://public.example/"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_unresolvable_host_is_rejected(monkeypatch):
|
||||||
|
def boom(host, port, *args, **kwargs):
|
||||||
|
raise socket.gaierror("no such host")
|
||||||
|
|
||||||
|
monkeypatch.setattr(safehttp_transport.socket, "getaddrinfo", boom)
|
||||||
|
transport = AsyncSafeTransport()
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
transport._validate(_request("https://nope.invalid/"))
|
||||||
|
|
||||||
|
|
||||||
|
class _LocalServer:
|
||||||
|
def __init__(self):
|
||||||
|
class Handler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
self.send_response(200)
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b"OK")
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
self.send_response(200)
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b"OK")
|
||||||
|
|
||||||
|
def log_message(self, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
self.server = http.server.HTTPServer(("127.0.0.1", 0), Handler)
|
||||||
|
self.port = self.server.server_address[1]
|
||||||
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
self.thread.start()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
self.server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_transport_pins_connection_to_validated_ip(monkeypatch):
|
||||||
|
with _LocalServer() as server:
|
||||||
|
# host does not really resolve to localhost; the resolver + pin make it so
|
||||||
|
_patch_resolver(monkeypatch, ["127.0.0.1"])
|
||||||
|
transport = AsyncSafeTransport(allow_hosts=["pinned.example"])
|
||||||
|
async with httpx.AsyncClient(transport=transport) as client:
|
||||||
|
resp = await client.get(f"http://pinned.example:{server.port}/")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert resp.text == "OK"
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_post_blocks_loopback_by_default():
|
||||||
|
with _LocalServer() as server:
|
||||||
|
with pytest.raises(InvalidDomainError):
|
||||||
|
safehttp.post(f"http://127.0.0.1:{server.port}/", json={"x": 1})
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_post_allows_configured_host():
|
||||||
|
with _LocalServer() as server:
|
||||||
|
resp = safehttp.post(
|
||||||
|
f"http://127.0.0.1:{server.port}/",
|
||||||
|
json={"x": 1},
|
||||||
|
allow_hosts=["127.0.0.1"],
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
Reference in New Issue
Block a user