refactor: unify authoring apps into openedx_content (#454)

This commit is a major refactoring that does the following:

- All existing authoring apps (backup_restore, collections, components,
  contents, publishing, sections, subsections, units) are unified into
  the new openedx_content app.
- The logical separation between the previous apps are preserved as
  packages in a new "applets" subpackage inside of openedx_content.
  Each applets subpackage has its own models.py and api.py modules.
- Skeletons of the old apps are preserved in the new "backcompat"
  package inside of openedx_content. This is done to preserve database
  migration compatibility.
- Increments the version to 0.31.0

The motivation for this change is to make it easier to refactor apps
going forward. Having entirely separate apps makes it difficult to make
any changes that involve moving models across apps. This has come up a
couple of times already, when trying to extract container logic out of
publishing and when trying to rename the "contents" app to "media".

The name change away from "authoring" also reflects that we intend to
use these APIs and models on the "learning" side of things as well, e.g.
when reading content for rendering to a student. In the longer term, we
will probably make openedx_content a top-level package in this repo, but
that would be a later step.

Most of this commit is just shuffling things around and renaming
references, but the truly complicated bits are around the sequencing of
database migrations, particularly the ones removing model state from the
old apps, and transferring that state into the new openedx_content app
without changing the actual database state.

All this is explained in greater detail in the following ADR:

  docs/decisions/0020-merge-authoring-apps-into-openedx-content.rst

Author: ormsbee

.annotation_safe_list.yml

@@ -19,43 +19,43 @@ auth.User:
   ".. pii_retirement": "consumer_api"
 contenttypes.ContentType:
   ".. no_pii:": "This model has no PII"
-oel_collections.Collection:
+openedx_content.Collection:
   ".. no_pii:": "This model has no PII"
-oel_collections.CollectionPublishableEntity:
+openedx_content.CollectionPublishableEntity:
   ".. no_pii:": "This model has no PII"
-oel_components.Component:
+openedx_content.Component:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentType:
+openedx_content.ComponentType:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentVersion:
+openedx_content.ComponentVersion:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentVersionContent:
+openedx_content.ComponentVersionContent:
   ".. no_pii:": "This model has no PII"
-oel_contents.Content:
+openedx_content.Content:
   ".. no_pii:": "This model has no PII"
-oel_contents.MediaType:
+openedx_content.MediaType:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Container:
+openedx_content.Container:
   ".. no_pii:": "This model has no PII"
-oel_publishing.ContainerVersion:
+openedx_content.ContainerVersion:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Draft:
+openedx_content.Draft:
   ".. no_pii:": "This model has no PII"
-oel_publishing.EntityList:
+openedx_content.EntityList:
   ".. no_pii:": "This model has no PII"
-oel_publishing.EntityListRow:
+openedx_content.EntityListRow:
   ".. no_pii:": "This model has no PII"
-oel_publishing.LearningPackage:
+openedx_content.LearningPackage:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishLog:
+openedx_content.PublishLog:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishLogRecord:
+openedx_content.PublishLogRecord:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishableEntity:
+openedx_content.PublishableEntity:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishableEntityVersion:
+openedx_content.PublishableEntityVersion:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Published:
+openedx_content.Published:
   ".. no_pii:": "This model has no PII"
 oel_tagging.ObjectTag:
   ".. no_pii:": "This model has no PII"
@@ -65,17 +65,17 @@ oel_tagging.TagImportTask:
   ".. no_pii:": "This model has no PII"
 oel_tagging.Taxonomy:
   ".. no_pii:": "This model has no PII"
-oel_sections.Section:
+openedx_content.Section:
   ".. no_pii:": "This model has no PII"
-oel_sections.SectionVersion:
+openedx_content.SectionVersion:
   ".. no_pii:": "This model has no PII"
-oel_subsections.Subsection:
+openedx_content.Subsection:
   ".. no_pii:": "This model has no PII"
-oel_subsections.SubsectionVersion:
+openedx_content.SubsectionVersion:
   ".. no_pii:": "This model has no PII"
-oel_units.Unit:
+openedx_content.Unit:
   ".. no_pii:": "This model has no PII"
-oel_units.UnitVersion:
+openedx_content.UnitVersion:
   ".. no_pii:": "This model has no PII"
 social_django.Association:
   ".. no_pii:": "This model has no PII"

.gitignore

@@ -75,7 +75,12 @@ venv/
 !.vscode/settings.json.example
 
 # Media files (for uploads)
-media/
+/media/
 
 # Media files generated during test runs
-test_media/
+/test_media/
+
+# uv stuff
+.lock
+CACHEDIR.TAG
+pyvenv.cfg

.importlinter

@@ -36,24 +36,24 @@ layers=
     openedx_learning.api.authoring
 
     # The "backup_restore" app handle the new export and import mechanism.
-    openedx_learning.apps.authoring.backup_restore
+    openedx_learning.apps.openedx_content.applets.backup_restore
 
     # The "components" app is responsible for storing versioned Components,
     # which is Open edX Studio terminology maps to things like individual
     # Problems, Videos, and blocks of HTML text. This is also the type we would
     # associate with a single "leaf" XBlock–one that is not a container type and
     # has no child elements.
-    openedx_learning.apps.authoring.components
+    openedx_learning.apps.openedx_content.applets.components
 
     # The "contents" app stores the simplest pieces of binary and text data,
     # without versioning information. These belong to a single Learning Package.
-    openedx_learning.apps.authoring.contents
+    openedx_learning.apps.openedx_content.applets.contents
 
     # The "collections" app stores arbitrary groupings of PublishableEntities.
     # Its only dependency should be the publishing app.
-    openedx_learning.apps.authoring.collections
+    openedx_learning.apps.openedx_content.applets.collections
 
     # The lowest layer is "publishing", which holds the basic primitives needed
     # to create Learning Packages and manage the draft and publish states for
     # various types of content.
-    openedx_learning.apps.authoring.publishing
+    openedx_learning.apps.openedx_content.applets.publishing

docs/decisions/0020-merge-authoring-apps-into-openedx-content.rst

@@ -0,0 +1,87 @@
+20. Merge authoring apps into openedx_content (using Applets)
+=============================================================
+
+Context
+-------
+
+Up to this point, Learning Core has used many small apps with a narrow focus (e.g. ``components``, ``collections``, etc.) in order to make each individual app simpler to reason about. This has been useful overall, but it has made refactoring more cumbersome. For instance:
+
+#. Moving models between apps is tricky, requiring the use of Django's ``SeparateDatabaseAndState`` functionality to fake a deletion in one app and a creation in another without actually altering the database. Moving models also introduces tricky dependencies with respect to migration ordering (described in more detail later in this document). We encountered this when considering how to extract container functionality out of the ``publishing`` app.
+#. Renaming an app is also cumbersome, because the process requires creating a new app and transitioning the models over. This came up when trying to rename the ``contents`` app to ``media``.
+
+There have also been minor inconveniences, like having a long list of ``INSTALLED_APPS`` to maintain in openedx-platform over time, or not having these tables easily grouped together in the Django admin interface.
+
+Decisions
+---------
+
+1. Single openedx_content App
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All existing authoring apps will be merged into one Django app named ``openedx_content``. Some consequences of this decision:
+
+- The tables will be renamed to have the ``openedx_content`` label prefix.
+- All management commands will be moved to the ``openedx_content`` app.
+
+2. Logical Separation via Applets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We will continue to keep internal API boundaries by using a new "Applets" convention. A Django applet is made to look like a miniature Django app, with its own ``models.py``, ``api.py``, and potentially other modules. The modules for the old authoring apps will be copied into various subpackages of ``openedx_content.applets``, such as ``openedx_content.applets.publishing``. Applets should respect each others' API boundaries, and never directly query models across applets. As before, we will use Import Linter to enforce dependency ordering.
+
+3. Package Restructuring
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+In one pull request, we are going to:
+
+#. Rename the ``openedx_learning.apps.authoring`` package to be ``openedx_learning.apps.openedx_content``. (Note: We have discussed eventually moving this to a top level ``openedx_content`` app instead of ``openedx_learning.apps.openedx_content``, but that will happen at a later time.)
+#. Create bare shells of the existing ``authoring`` apps (``backup_restore``, ``collections``, ``components``, ``contents``, ``publishing``, ``sections``, ``subsections``, ``units``), and move them to the ``openedx_learning.apps.openedx_content.backcompat`` package. These shells will have an ``apps.py`` file, the ``migrations`` package for each existing app, and in some cases a minimal ``models.py`` that will hold the skeletons of a handful of models. This will allow for a smooth schema migration to transition the models from these individual apps to ``openedx_content``.
+#. Move the actual models files and API logic for our existing authoring apps to the ``openedx_learning.apps.openedx_content.applets`` package.
+#. Convert the top level ``openedx_learning.apps.openedx_content`` package to be a Django app. The top level ``admin.py``, ``api.py``, and ``models.py`` modules will do wildcard imports from the corresponding modules across all applet packages.
+#. Test packages will also be updated to follow the new structure.
+
+4. Database Migration Specifics
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When Django runs migrations, it both:
+
+* Calculates an ephemeral logical model **state**, based on the contents of the Python migration files and the ``django_migration`` database table, which indicates the list of migrations that have been "run".
+* Actually executes the migration operations on the app **database** tables as each migration is "run".
+
+We are going to take advantage of the fact that these two can be separated using the ``SeparateDatabaseAndState`` operation. We will use this to remove the model state from the old authoring apps and create the model state in the new ``openedx_content`` app without having to run database operations.
+
+There are a few high level constraints that we have to consider:
+
+#. Existing openedx-platform migrations should not be modified. Existing openedx-platform migrations should remain unchanged. This is important to make sure that we do not introduce ordering inconsistencies for sites that have already run migrations for the old apps and are upgrading to a new release (e.g. Verawood).
+#. The openedx-learning repo should not have any dependencies on openedx-platform migrations, because our dependencies strictly go in the other direction: openedx-platform calls openedx-learning, not the other way around. Furthermore, openedx-learning will often be run without openedx-platform, such as for local development or during CI.
+#. Two of the openedx-platform apps that have foreign keys to openedx-learning models are only in Studio's INSTALLED_APPS (``contentstore`` and ``modulestore_migrator``), while ``content_libraries`` is installed in both Studio and LMS. Migrations may be run for LMS or Studio first, depending on the user and environment. Tutor runs LMS first, but we can't assume that will always be true.
+#. We must support people who are installing from scratch, those who are upgrading from the Ulmo release, as well as those who are running off of the master branch of openedx-platform.
+
+Therefore, the migrations will happen in the following order:
+
+#. All pre-existing ``backcompat.*`` apps migrations run as before.
+#. New ``backcompat.*`` apps migrations that drop most model state, but leave the database unchanged.
+#. The first ``openedx_content`` migration creates logical models without any database changes.
+#. The second ``openedx_content`` migration renames the underlying tables.
+#. Each the ``openedx-platform`` apps that had foreign keys to the old authoring apps will get a new migration that switches those foreign keys to point to ``openedx_content`` apps instead. These are: ``content_libraries``, ``contentstore``, and ``modulestore_migrator``.
+#. The above ``openedx-platform`` apps will also get a squashed migration that sets them up to point to the new ``openedx_content`` models directly.
+
+The tricky part is that all the ``opendx-learning`` migrations will run before any of the ``openedx-platform`` migrations run. We can't force it to do otherwise without making ``openedx-learning`` aware of ``opendx-platform``, and we explicitly want to avoid that. This makes things tricky with respect to the model state dependencies. There are two scenarios we have to worry about:
+
+Migration from Scrach
+  The ``openedx-platform`` apps will each run the squashed migration that jumps straight to making foreign keys against the new ``openedx_content`` models, so the fact that the old authoring app models have been removed and the tables have been renamed doesn't matter.
+
+Migration from Ulmo/master
+  No actual database operations have to happen here, as the keys were already created earlier. That being said, the migration framework will error out if the state of the old app models that it had foreign keys to have been dropped entirely. That's why the bare skeletons of those old models are preserved in the ``backcompat`` app models files, along with their primary key field. Everything else can be dropped from the state point of view—though again, we're not modifying database state in this operation.
+
+  The main downside of this approach is that it may break migrations for developers if they have a months old dev database that is in an in-between release state, e.g. after some ``modulestore_migrations`` referencing the old app mdoels were run, but before the most recent ``modulestore_migrations`` creating foreign keys to those models. No production environment is expected to deploy like this, so this would mainly impact developers. The workaround to this would be to run ``python manage.py lms migrate openedx_content 0001`` to rename the tables to their old values, then run the failing ``contentstore``, ``content_libraries``, or ``modulestore_migrator`` migrations again.
+
+5. Rejected Alternatives
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+An earlier attempt at this tried to solve the migration ordering issue by dynamically injecting migration dependencies into the second ``openedx_content`` migration module during the app config ``ready()`` initialization. This was later abandoned because it didn't solve the problem of CMS vs LMS differences in ``INSTALLED_APPS``, so the ordering could still get corrupted unless we added those apps to LMS—which would have introduced more risk.
+
+It's also worth noting that Django startup checks will fail if it detects that multiple models point to the same table. This is why we rename the tables for the ``openedx_contents`` models, and leave the skeleton models in ``backcompat`` apps pointing to the old table names (which no longer really exist in the database once these migrations run).
+
+6. The Bigger Picture
+~~~~~~~~~~~~~~~~~~~~~
+
+This overall refactoring means that the ``openedx_content`` Django app corresponds to a Subdomain in Domain Driven Design terminology, with each applet being a Bounded Context. We call these "Applets" instead of "Bounded Contexts" because we don't want it to get confused for Django's notion of Contexts and Context Processors (or Python's notion of Context Managers).

olx_importer/management/commands/load_components.py

@@ -28,9 +28,9 @@
 from django.db import transaction
 
 # Model references to remove
-from openedx_learning.apps.authoring.components import api as components_api
-from openedx_learning.apps.authoring.contents import api as contents_api
-from openedx_learning.apps.authoring.publishing import api as publishing_api
+from openedx_learning.apps.openedx_content.applets.components import api as components_api
+from openedx_learning.apps.openedx_content.applets.contents import api as contents_api
+from openedx_learning.apps.openedx_content.applets.publishing import api as publishing_api
 
 SUPPORTED_TYPES = ["problem", "video", "html"]
 logger = logging.getLogger(__name__)

openedx_learning/__init__.py

@@ -2,4 +2,4 @@
 Open edX Learning ("Learning Core").
 """
 
-__version__ = "0.30.2"
+__version__ = "0.31.0"

openedx_learning/api/authoring.py

@@ -2,21 +2,14 @@
 This is the public API for content authoring in Learning Core.
 
 This is the single ``api`` module that code outside of the
-``openedx_learning.apps.authoring.*`` package should import from. It will
+``openedx_learning.apps.openedx_content.*`` package should import from. It will
 re-export the public functions from all api.py modules of all authoring apps. It
 may also implement its own convenience APIs that wrap calls to multiple app
 APIs.
 """
 # These wildcard imports are okay because these api modules declare __all__.
 # pylint: disable=wildcard-import
-from ..apps.authoring.backup_restore.api import *
-from ..apps.authoring.collections.api import *
-from ..apps.authoring.components.api import *
-from ..apps.authoring.contents.api import *
-from ..apps.authoring.publishing.api import *
-from ..apps.authoring.sections.api import *
-from ..apps.authoring.subsections.api import *
-from ..apps.authoring.units.api import *
+from ..apps.openedx_content.api import *
 
 # This was renamed after the authoring API refactoring pushed this and other
 # app APIs into the openedx_learning.api.authoring module. Here I'm aliasing to

openedx_learning/api/authoring_models.py

@@ -7,10 +7,10 @@
 """
 # These wildcard imports are okay because these modules declare __all__.
 # pylint: disable=wildcard-import
-from ..apps.authoring.collections.models import *
-from ..apps.authoring.components.models import *
-from ..apps.authoring.contents.models import *
-from ..apps.authoring.publishing.models import *
-from ..apps.authoring.sections.models import *
-from ..apps.authoring.subsections.models import *
-from ..apps.authoring.units.models import *
+from ..apps.openedx_content.applets.collections.models import *
+from ..apps.openedx_content.applets.components.models import *
+from ..apps.openedx_content.applets.contents.models import *
+from ..apps.openedx_content.applets.publishing.models import *
+from ..apps.openedx_content.applets.sections.models import *
+from ..apps.openedx_content.applets.subsections.models import *
+from ..apps.openedx_content.applets.units.models import *

openedx_learning/api/django.py

@@ -0,0 +1,24 @@
+"""
+Module for parts of the Learning Core API that exist to make it easier to use in
+Django projects.
+"""
+
+
+def openedx_learning_apps_to_install():
+    """
+    Return all app names for appending to INSTALLED_APPS.
+
+    This function exists to better insulate edx-platform and potential plugins
+    over time, as we eventually plan to remove the backcompat apps.
+    """
+    return [
+        "openedx_learning.apps.openedx_content",
+        "openedx_learning.apps.openedx_content.backcompat.backup_restore",
+        "openedx_learning.apps.openedx_content.backcompat.collections",
+        "openedx_learning.apps.openedx_content.backcompat.components",
+        "openedx_learning.apps.openedx_content.backcompat.contents",
+        "openedx_learning.apps.openedx_content.backcompat.publishing",
+        "openedx_learning.apps.openedx_content.backcompat.sections",
+        "openedx_learning.apps.openedx_content.backcompat.subsections",
+        "openedx_learning.apps.openedx_content.backcompat.units",
+    ]