Remove all sqlalchemy lazy-loading from app (#2260)

* Remove some implicit lazy-loads from user serialization

* implement full backup restore across different database versions

* rework all custom getter dicts to not leak lazy loads

* remove some occurances of lazy-loading

* remove a lot of lazy loading from recipes

* add more eager loading
remove loading options from repository
remove raiseload for checking

* fix failing test

* do not apply loader options for paging counts

* try using selectinload a bit more instead of joinedload

* linter fixes
This commit is contained in:
Sören
2023-03-24 17:27:26 +01:00
committed by GitHub
parent fae62ecb19
commit 4b426ddf2f
23 changed files with 351 additions and 142 deletions

View File

@@ -5,6 +5,7 @@ from typing import Protocol, TypeVar
from humps.main import camelize
from pydantic import UUID4, BaseModel
from sqlalchemy.orm.interfaces import LoaderOption
T = TypeVar("T", bound=BaseModel)
@@ -54,6 +55,10 @@ class MealieModel(BaseModel):
if field in self.__fields__ and (val is not None or replace_null):
setattr(self, field, val)
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return []
class HasUUID(Protocol):
id: UUID4

View File

@@ -1,10 +1,13 @@
from pydantic import UUID4, validator
from slugify import slugify
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.schema._mealie import MealieModel
from mealie.schema.recipe.recipe import RecipeSummary, RecipeTool
from mealie.schema.response.pagination import PaginationBase
from ...db.models.group import CookBook
from ..recipe.recipe_category import CategoryBase, TagBase
@@ -51,6 +54,10 @@ class ReadCookBook(UpdateCookBook):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(CookBook.categories), joinedload(CookBook.tags), joinedload(CookBook.tools)]
class CookBookPagination(PaginationBase):
items: list[ReadCookBook]

View File

@@ -0,0 +1,33 @@
from collections.abc import Callable, Mapping
from typing import Any
from pydantic.utils import GetterDict
class CustomGetterDict(GetterDict):
transformations: Mapping[str, Callable[[Any], Any]]
def get(self, key: Any, default: Any = None) -> Any:
# Transform extras into key-value dict
if key in self.transformations:
value = super().get(key, default)
return self.transformations[key](value)
# Keep all other fields as they are
else:
return super().get(key, default)
class ExtrasGetterDict(CustomGetterDict):
transformations = {"extras": lambda value: {x.key_name: x.value for x in value}}
class GroupGetterDict(CustomGetterDict):
transformations = {"group": lambda value: value.name}
class UserGetterDict(CustomGetterDict):
transformations = {
"group": lambda value: value.name,
"favorite_recipes": lambda value: [x.slug for x in value],
}

View File

@@ -1,5 +1,8 @@
from pydantic import UUID4, NoneStr
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.group import GroupEventNotifierModel
from mealie.schema._mealie import MealieModel
from mealie.schema.response.pagination import PaginationBase
@@ -86,6 +89,10 @@ class GroupEventNotifierOut(MealieModel):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(GroupEventNotifierModel.options)]
class GroupEventPagination(PaginationBase):
items: list[GroupEventNotifierOut]

View File

@@ -4,11 +4,19 @@ from datetime import datetime
from fractions import Fraction
from pydantic import UUID4, validator
from pydantic.utils import GetterDict
from sqlalchemy.orm import joinedload, selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.group.shopping_list import ShoppingList, ShoppingListItem
from mealie.db.models.group import (
ShoppingList,
ShoppingListItem,
ShoppingListMultiPurposeLabel,
ShoppingListRecipeReference,
)
from mealie.db.models.recipe import IngredientFoodModel, RecipeModel
from mealie.schema._mealie import MealieModel
from mealie.schema._mealie.types import NoneFloat
from mealie.schema.getter_dict import ExtrasGetterDict
from mealie.schema.labels.multi_purpose_label import MultiPurposeLabelSummary
from mealie.schema.recipe.recipe import RecipeSummary
from mealie.schema.recipe.recipe_ingredient import (
@@ -171,13 +179,18 @@ class ShoppingListItemOut(ShoppingListItemBase):
class Config:
orm_mode = True
getter_dict = ExtrasGetterDict
@classmethod
def getter_dict(cls, name_orm: ShoppingListItem):
return {
**GetterDict(name_orm),
"extras": {x.key_name: x.value for x in name_orm.extras},
}
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(ShoppingListItem.extras),
selectinload(ShoppingListItem.food).joinedload(IngredientFoodModel.extras),
selectinload(ShoppingListItem.food).joinedload(IngredientFoodModel.label),
joinedload(ShoppingListItem.label),
joinedload(ShoppingListItem.unit),
selectinload(ShoppingListItem.recipe_references),
]
class ShoppingListItemsCollectionOut(MealieModel):
@@ -204,6 +217,10 @@ class ShoppingListMultiPurposeLabelOut(ShoppingListMultiPurposeLabelUpdate):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(ShoppingListMultiPurposeLabel.label)]
class ShoppingListItemPagination(PaginationBase):
items: list[ShoppingListItemOut]
@@ -229,6 +246,14 @@ class ShoppingListRecipeRefOut(MealieModel):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(ShoppingListRecipeReference.recipe).joinedload(RecipeModel.recipe_category),
selectinload(ShoppingListRecipeReference.recipe).joinedload(RecipeModel.tags),
selectinload(ShoppingListRecipeReference.recipe).joinedload(RecipeModel.tools),
]
class ShoppingListSave(ShoppingListCreate):
group_id: UUID4
@@ -241,13 +266,23 @@ class ShoppingListSummary(ShoppingListSave):
class Config:
orm_mode = True
getter_dict = ExtrasGetterDict
@classmethod
def getter_dict(cls, name_orm: ShoppingList):
return {
**GetterDict(name_orm),
"extras": {x.key_name: x.value for x in name_orm.extras},
}
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(ShoppingList.extras),
selectinload(ShoppingList.recipe_references)
.joinedload(ShoppingListRecipeReference.recipe)
.joinedload(RecipeModel.recipe_category),
selectinload(ShoppingList.recipe_references)
.joinedload(ShoppingListRecipeReference.recipe)
.joinedload(RecipeModel.tags),
selectinload(ShoppingList.recipe_references)
.joinedload(ShoppingListRecipeReference.recipe)
.joinedload(RecipeModel.tools),
selectinload(ShoppingList.label_settings).joinedload(ShoppingListMultiPurposeLabel.label),
]
class ShoppingListPagination(PaginationBase):
@@ -265,13 +300,33 @@ class ShoppingListOut(ShoppingListUpdate):
class Config:
orm_mode = True
getter_dict = ExtrasGetterDict
@classmethod
def getter_dict(cls, name_orm: ShoppingList):
return {
**GetterDict(name_orm),
"extras": {x.key_name: x.value for x in name_orm.extras},
}
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(ShoppingList.extras),
selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.extras),
selectinload(ShoppingList.list_items)
.joinedload(ShoppingListItem.food)
.joinedload(IngredientFoodModel.extras),
selectinload(ShoppingList.list_items)
.joinedload(ShoppingListItem.food)
.joinedload(IngredientFoodModel.label),
selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.label),
selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.unit),
selectinload(ShoppingList.list_items).joinedload(ShoppingListItem.recipe_references),
selectinload(ShoppingList.recipe_references)
.joinedload(ShoppingListRecipeReference.recipe)
.joinedload(RecipeModel.recipe_category),
selectinload(ShoppingList.recipe_references)
.joinedload(ShoppingListRecipeReference.recipe)
.joinedload(RecipeModel.tags),
selectinload(ShoppingList.recipe_references)
.joinedload(ShoppingListRecipeReference.recipe)
.joinedload(RecipeModel.tools),
selectinload(ShoppingList.label_settings).joinedload(ShoppingListMultiPurposeLabel.label),
]
class ShoppingListAddRecipeParams(MealieModel):

View File

@@ -3,7 +3,11 @@ from enum import Enum
from uuid import UUID
from pydantic import validator
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.group import GroupMealPlan
from mealie.db.models.recipe import RecipeModel
from mealie.schema._mealie import MealieModel
from mealie.schema.recipe.recipe import RecipeSummary
from mealie.schema.response.pagination import PaginationBase
@@ -57,6 +61,14 @@ class ReadPlanEntry(UpdatePlanEntry):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(GroupMealPlan.recipe).joinedload(RecipeModel.recipe_category),
selectinload(GroupMealPlan.recipe).joinedload(RecipeModel.tags),
selectinload(GroupMealPlan.recipe).joinedload(RecipeModel.tools),
]
class PlanEntryPagination(PaginationBase):
items: list[ReadPlanEntry]

View File

@@ -2,7 +2,10 @@ import datetime
from enum import Enum
from pydantic import UUID4
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.group import GroupMealPlanRules
from mealie.schema._mealie import MealieModel
from mealie.schema.response.pagination import PaginationBase
@@ -65,6 +68,10 @@ class PlanRulesOut(PlanRulesSave):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(GroupMealPlanRules.categories), joinedload(GroupMealPlanRules.tags)]
class PlanRulesPagination(PaginationBase):
items: list[PlanRulesOut]

View File

@@ -1,7 +1,5 @@
from pydantic.utils import GetterDict
from mealie.db.models.group.shopping_list import ShoppingList
from mealie.schema._mealie import MealieModel
from mealie.schema.getter_dict import GroupGetterDict
class ListItem(MealieModel):
@@ -25,10 +23,4 @@ class ShoppingListOut(ShoppingListIn):
class Config:
orm_mode = True
@classmethod
def getter_dict(cls, ormModel: ShoppingList):
return {
**GetterDict(ormModel),
"group": ormModel.group.name,
}
getter_dict = GroupGetterDict

View File

@@ -6,14 +6,22 @@ from typing import Any
from uuid import uuid4
from pydantic import UUID4, BaseModel, Field, validator
from pydantic.utils import GetterDict
from slugify import slugify
from sqlalchemy.orm import joinedload, selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.core.config import get_app_dirs
from mealie.db.models.recipe.recipe import RecipeModel
from mealie.schema._mealie import MealieModel
from mealie.schema.response.pagination import PaginationBase
from ...db.models.recipe import (
IngredientFoodModel,
RecipeComment,
RecipeIngredientModel,
RecipeInstruction,
RecipeModel,
)
from ..getter_dict import ExtrasGetterDict
from .recipe_asset import RecipeAsset
from .recipe_comments import RecipeCommentOut
from .recipe_notes import RecipeNote
@@ -147,16 +155,7 @@ class Recipe(RecipeSummary):
class Config:
orm_mode = True
@classmethod
def getter_dict(cls, name_orm: RecipeModel):
return {
**GetterDict(name_orm),
# "recipe_ingredient": [x.note for x in name_orm.recipe_ingredient],
# "recipe_category": [x.name for x in name_orm.recipe_category],
# "tags": [x.name for x in name_orm.tags],
"extras": {x.key_name: x.value for x in name_orm.extras},
}
getter_dict = ExtrasGetterDict
@validator("slug", always=True, pre=True, allow_reuse=True)
def validate_slug(slug: str, values): # type: ignore
@@ -199,6 +198,29 @@ class Recipe(RecipeSummary):
return uuid4()
return user_id
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(RecipeModel.assets),
selectinload(RecipeModel.comments).joinedload(RecipeComment.user),
selectinload(RecipeModel.extras),
joinedload(RecipeModel.recipe_category),
selectinload(RecipeModel.tags),
selectinload(RecipeModel.tools),
selectinload(RecipeModel.recipe_ingredient).joinedload(RecipeIngredientModel.unit),
selectinload(RecipeModel.recipe_ingredient)
.joinedload(RecipeIngredientModel.food)
.joinedload(IngredientFoodModel.extras),
selectinload(RecipeModel.recipe_ingredient)
.joinedload(RecipeIngredientModel.food)
.joinedload(IngredientFoodModel.label),
selectinload(RecipeModel.recipe_instructions).joinedload(RecipeInstruction.ingredient_references),
joinedload(RecipeModel.nutrition),
joinedload(RecipeModel.settings),
# for whatever reason, joinedload can mess up the order here, so use selectinload just this once
selectinload(RecipeModel.notes),
]
class RecipeLastMade(BaseModel):
timestamp: datetime.datetime

View File

@@ -1,6 +1,8 @@
from pydantic import UUID4
from pydantic.utils import GetterDict
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.recipe import RecipeModel, Tag
from mealie.schema._mealie import MealieModel
@@ -19,12 +21,6 @@ class CategoryBase(CategoryIn):
class Config:
orm_mode = True
@classmethod
def getter_dict(_cls, name_orm):
return {
**GetterDict(name_orm),
}
class CategoryOut(CategoryBase):
slug: str
@@ -62,7 +58,13 @@ class TagOut(TagSave):
class RecipeTagResponse(RecipeCategoryResponse):
pass
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(Tag.recipes).joinedload(RecipeModel.recipe_category),
selectinload(Tag.recipes).joinedload(RecipeModel.tags),
selectinload(Tag.recipes).joinedload(RecipeModel.tools),
]
from mealie.schema.recipe.recipe import RecipeSummary # noqa: E402

View File

@@ -1,7 +1,10 @@
from datetime import datetime
from pydantic import UUID4
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.recipe import RecipeComment
from mealie.schema._mealie import MealieModel
from mealie.schema.response.pagination import PaginationBase
@@ -40,6 +43,10 @@ class RecipeCommentOut(RecipeCommentCreate):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(RecipeComment.user)]
class RecipeCommentPagination(PaginationBase):
items: list[RecipeCommentOut]

View File

@@ -2,14 +2,16 @@ from __future__ import annotations
import datetime
import enum
from typing import Any
from uuid import UUID, uuid4
from pydantic import UUID4, Field, validator
from pydantic.utils import GetterDict
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.recipe import IngredientFoodModel
from mealie.schema._mealie import MealieModel
from mealie.schema._mealie.types import NoneFloat
from mealie.schema.getter_dict import ExtrasGetterDict
from mealie.schema.response.pagination import PaginationBase
INGREDIENT_QTY_PRECISION = 3
@@ -37,19 +39,12 @@ class IngredientFood(CreateIngredientFood):
update_at: datetime.datetime | None
class Config:
class _FoodGetter(GetterDict):
def get(self, key: Any, default: Any = None) -> Any:
# Transform extras into key-value dict
if key == "extras":
value = super().get(key, default)
return {x.key_name: x.value for x in value}
# Keep all other fields as they are
else:
return super().get(key, default)
orm_mode = True
getter_dict = _FoodGetter
getter_dict = ExtrasGetterDict
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(IngredientFoodModel.extras), joinedload(IngredientFoodModel.label)]
class IngredientFoodPagination(PaginationBase):

View File

@@ -1,9 +1,12 @@
from datetime import datetime, timedelta
from pydantic import UUID4, Field
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.schema._mealie import MealieModel
from ...db.models.recipe import RecipeIngredientModel, RecipeInstruction, RecipeModel, RecipeShareTokenModel
from .recipe import Recipe
@@ -33,3 +36,26 @@ class RecipeShareToken(RecipeShareTokenSummary):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.recipe_category),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.tags),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.tools),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.nutrition),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.settings),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.assets),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.notes),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.extras),
selectinload(RecipeShareTokenModel.recipe).joinedload(RecipeModel.comments),
selectinload(RecipeShareTokenModel.recipe)
.joinedload(RecipeModel.recipe_instructions)
.joinedload(RecipeInstruction.ingredient_references),
selectinload(RecipeShareTokenModel.recipe)
.joinedload(RecipeModel.recipe_ingredient)
.joinedload(RecipeIngredientModel.unit),
selectinload(RecipeShareTokenModel.recipe)
.joinedload(RecipeModel.recipe_ingredient)
.joinedload(RecipeIngredientModel.food),
]

View File

@@ -1,7 +1,11 @@
from pydantic import UUID4
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.schema._mealie import MealieModel
from ...db.models.recipe import RecipeModel, Tool
class RecipeToolCreate(MealieModel):
name: str
@@ -21,12 +25,20 @@ class RecipeToolOut(RecipeToolCreate):
class RecipeToolResponse(RecipeToolOut):
recipes: list["Recipe"] = []
recipes: list["RecipeSummary"] = []
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(Tool.recipes).joinedload(RecipeModel.recipe_category),
selectinload(Tool.recipes).joinedload(RecipeModel.tags),
selectinload(Tool.recipes).joinedload(RecipeModel.tools),
]
from .recipe import Recipe # noqa: E402
from .recipe import RecipeSummary # noqa: E402
RecipeToolResponse.update_forward_refs()

View File

@@ -3,7 +3,10 @@ import enum
from pydantic import Field
from pydantic.types import UUID4
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.db.models.group import ReportModel
from mealie.schema._mealie import MealieModel
@@ -53,3 +56,7 @@ class ReportOut(ReportSummary):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(ReportModel.entries)]

View File

@@ -5,7 +5,8 @@ from uuid import UUID
from pydantic import UUID4, Field, validator
from pydantic.types import constr
from pydantic.utils import GetterDict
from sqlalchemy.orm import joinedload, selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.core.config import get_app_dirs, get_app_settings
from mealie.db.models.users import User
@@ -15,6 +16,9 @@ from mealie.schema.group.group_preferences import ReadGroupPreferences
from mealie.schema.recipe import RecipeSummary
from mealie.schema.response.pagination import PaginationBase
from ...db.models.group import Group
from ...db.models.recipe import RecipeModel
from ..getter_dict import GroupGetterDict, UserGetterDict
from ..recipe import CategoryBase
DEFAULT_INTEGRATION_ID = "generic"
@@ -78,19 +82,8 @@ class UserBase(MealieModel):
can_organize: bool = False
class Config:
class _UserGetter(GetterDict):
def get(self, key: Any, default: Any = None) -> Any:
# Transform extras into key-value dict
if key == "group":
value = super().get(key, default)
return value.group.name
# Keep all other fields as they are
else:
return super().get(key, default)
orm_mode = True
getter_dict = _UserGetter
getter_dict = GroupGetterDict
schema_extra = {
"example": {
@@ -118,13 +111,11 @@ class UserOut(UserBase):
class Config:
orm_mode = True
@classmethod
def getter_dict(cls, ormModel: User):
return {
**GetterDict(ormModel),
"group": ormModel.group.name,
"favorite_recipes": [x.slug for x in ormModel.favorite_recipes],
}
getter_dict = UserGetterDict
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(User.group), joinedload(User.favorite_recipes), joinedload(User.tokens)]
class UserPagination(PaginationBase):
@@ -136,13 +127,16 @@ class UserFavorites(UserBase):
class Config:
orm_mode = True
getter_dict = GroupGetterDict
@classmethod
def getter_dict(cls, ormModel: User):
return {
**GetterDict(ormModel),
"group": ormModel.group.name,
}
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
joinedload(User.group),
selectinload(User.favorite_recipes).joinedload(RecipeModel.recipe_category),
selectinload(User.favorite_recipes).joinedload(RecipeModel.tags),
selectinload(User.favorite_recipes).joinedload(RecipeModel.tools),
]
class PrivateUser(UserOut):
@@ -175,6 +169,10 @@ class PrivateUser(UserOut):
def directory(self) -> Path:
return PrivateUser.get_directory(self.id)
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [joinedload(User.group), selectinload(User.favorite_recipes), joinedload(User.tokens)]
class UpdateGroup(GroupBase):
id: UUID4
@@ -211,6 +209,17 @@ class GroupInDB(UpdateGroup):
def exports(self) -> Path:
return GroupInDB.get_export_directory(self.id)
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
joinedload(Group.categories),
joinedload(Group.webhooks),
joinedload(Group.preferences),
selectinload(Group.users).joinedload(User.group),
selectinload(Group.users).joinedload(User.favorite_recipes),
selectinload(Group.users).joinedload(User.tokens),
]
class GroupPagination(PaginationBase):
items: list[GroupInDB]

View File

@@ -1,7 +1,10 @@
from pydantic import UUID4
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from mealie.schema._mealie import MealieModel
from ...db.models.users import PasswordResetModel, User
from .user import PrivateUser
@@ -33,3 +36,11 @@ class PrivatePasswordResetToken(SavePasswordResetToken):
class Config:
orm_mode = True
@classmethod
def loader_options(cls) -> list[LoaderOption]:
return [
selectinload(PasswordResetModel.user).joinedload(User.group),
selectinload(PasswordResetModel.user).joinedload(User.favorite_recipes),
selectinload(PasswordResetModel.user).joinedload(User.tokens),
]