diff --git a/AGENTS.md b/AGENTS.md index ce6e852..91166bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,7 +156,7 @@ TypeTrees describe how Unity objects are serialized (property names, types, offs ### File Formats - **Unity Archive** - Container format (AssetBundles, .data files). Can be mounted as virtual filesystem. - **SerializedFile** - Binary format storing Unity objects with TypeTree metadata. -- **Addressables BuildLayout** - JSON build report (buildlogreport.json, AddressablesReport.json) +- **JSON** - JSON files used for build reporting (schema found in UnityDataModels as C# data structures). ### Common Issues diff --git a/Analyzer/Resources/ContentLayoutViews.sql b/Analyzer/Resources/ContentLayoutViews.sql index 47dd247..134a356 100644 --- a/Analyzer/Resources/ContentLayoutViews.sql +++ b/Analyzer/Resources/ContentLayoutViews.sql @@ -53,7 +53,7 @@ SELECT artifact_index, content_hash, category, size, FROM content_layout_binary_artifacts; -- The data files (.resS/.resource) each serialized file uses, derived from the artifact graph. -CREATE VIEW IF NOT EXISTS content_layout_data_files_view AS +CREATE VIEW IF NOT EXISTS content_layout_resource_files_view AS SELECT f.file_index, f.content_hash || '.cf' AS filename, ra.category, rav.filename AS data_filename, ra.size FROM content_layout_serialized_files f diff --git a/Analyzer/SQLite/Handlers/BuildReportHandler.cs b/Analyzer/SQLite/Handlers/BuildReportHandler.cs index 78deb0f..e4a4d11 100644 --- a/Analyzer/SQLite/Handlers/BuildReportHandler.cs +++ b/Analyzer/SQLite/Handlers/BuildReportHandler.cs @@ -7,15 +7,18 @@ namespace UnityDataTools.Analyzer.SQLite.Handlers; public class BuildReportHandler : ISQLiteHandler { + private SqliteConnection m_Database; + private bool m_SchemaCreated; private SqliteCommand m_InsertCommand; private SqliteCommand m_InsertFileCommand; private SqliteCommand m_InsertArchiveContentCommand; public void Init(SqliteConnection db) { - using var command = db.CreateCommand(); - command.CommandText = Properties.Resources.BuildReport ?? throw new InvalidOperationException("BuildReport resource not found"); - command.ExecuteNonQuery(); + // The build_report tables are created lazily, on the first BuildReport object (see + // EnsureSchema), so a database analyzed without any build report is not cluttered with + // empty build_report tables. + m_Database = db; m_InsertCommand = db.CreateCommand(); m_InsertCommand.CommandText = @"INSERT INTO build_reports( @@ -70,8 +73,25 @@ public void Init(SqliteConnection db) m_InsertArchiveContentCommand.Parameters.Add("@archive_content", SqliteType.Text); } + // Creates the build_report schema on first use. Runs inside the current file's transaction, so + // it is committed together with the rows it is about to receive. + private void EnsureSchema(SqliteTransaction transaction) + { + if (m_SchemaCreated) + return; + + using var command = m_Database.CreateCommand(); + command.Transaction = transaction; + command.CommandText = Properties.Resources.BuildReport ?? throw new InvalidOperationException("BuildReport resource not found"); + command.ExecuteNonQuery(); + + m_SchemaCreated = true; + } + public void Process(Context ctx, long objectId, RandomAccessReader reader, out string name, out long streamDataSize) { + EnsureSchema(ctx.Transaction); + var buildReport = BuildReport.Read(reader); m_InsertCommand.Transaction = ctx.Transaction; m_InsertCommand.Parameters["@id"].Value = objectId; diff --git a/Analyzer/SQLite/Handlers/PackedAssetsHandler.cs b/Analyzer/SQLite/Handlers/PackedAssetsHandler.cs index 833cec4..72d7f34 100644 --- a/Analyzer/SQLite/Handlers/PackedAssetsHandler.cs +++ b/Analyzer/SQLite/Handlers/PackedAssetsHandler.cs @@ -9,6 +9,8 @@ namespace UnityDataTools.Analyzer.SQLite.Handlers; public class PackedAssetsHandler : ISQLiteHandler { + private SqliteConnection m_Database; + private bool m_SchemaCreated; private SqliteCommand m_InsertPackedAssetsCommand; private SqliteCommand m_InsertSourceAssetCommand; private SqliteCommand m_GetSourceAssetIdCommand; @@ -23,9 +25,10 @@ public class PackedAssetsHandler : ISQLiteHandler public void Init(SqliteConnection db) { - using var command = db.CreateCommand(); - command.CommandText = Properties.Resources.PackedAssets ?? throw new InvalidOperationException("PackedAssets resource not found"); - command.ExecuteNonQuery(); + // The build_report_packed_* tables and views are created lazily, on the first PackedAssets + // object (see EnsureSchema). A build report without any PackedAssets object (or a database + // analyzed without any build report) therefore does not get these tables. + m_Database = db; m_InsertPackedAssetsCommand = db.CreateCommand(); m_InsertPackedAssetsCommand.CommandText = @"INSERT INTO build_report_packed_assets( @@ -77,8 +80,28 @@ public void Init(SqliteConnection db) m_InsertTypeCommand.Parameters.Add("@name", SqliteType.Text); } + // Creates the build_report_packed_* schema on first use. Runs inside the current file's + // transaction, so it is committed together with the rows it is about to receive. Its views + // reference build_report_archive_contents (created by BuildReportHandler); SQLite does not + // resolve view references until query time, and every build report containing PackedAssets + // objects also contains the BuildReport object, so that table exists before any query. + private void EnsureSchema(SqliteTransaction transaction) + { + if (m_SchemaCreated) + return; + + using var command = m_Database.CreateCommand(); + command.Transaction = transaction; + command.CommandText = Properties.Resources.PackedAssets ?? throw new InvalidOperationException("PackedAssets resource not found"); + command.ExecuteNonQuery(); + + m_SchemaCreated = true; + } + public void Process(Context ctx, long objectId, RandomAccessReader reader, out string name, out long streamDataSize) { + EnsureSchema(ctx.Transaction); + var packedAssets = PackedAssets.Read(reader); m_InsertPackedAssetsCommand.Transaction = ctx.Transaction; diff --git a/Documentation/analyze-examples-contentlayout.md b/Documentation/analyze-examples-contentlayout.md index b7eb712..91ff1a4 100644 --- a/Documentation/analyze-examples-contentlayout.md +++ b/Documentation/analyze-examples-contentlayout.md @@ -44,7 +44,7 @@ SELECT v.* FROM closure INNER JOIN content_layout_serialized_files_view v USING Add up the `size` column of the result for the total load footprint (excluding data loaded on demand through loadables, and the `.resS`/`.resource` data files — join -`content_layout_data_files_view` to include those). +`content_layout_resource_files_view` to include those). ## All objects built from a source asset (layout + content) @@ -74,7 +74,7 @@ The `.resS`/`.resource` files holding the streamed texture/mesh and audio/video serialized file: ```sql -SELECT * FROM content_layout_data_files_view +SELECT * FROM content_layout_resource_files_view WHERE filename = 'c0152db4dd710be51b2decb997325f34.cf'; ``` diff --git a/Documentation/buildreport.md b/Documentation/buildreport.md index 5af322c..5c772cf 100644 --- a/Documentation/buildreport.md +++ b/Documentation/buildreport.md @@ -210,6 +210,8 @@ Build report data is stored in the following tables and views: The `build_reports` table contains primary build information. Additional tables store detailed content data. Views simplify queries by automatically joining tables, especially when working with multiple build reports. +These tables and views are created on demand, so a database analyzed without any build report does not contain them. The `build_reports`, `build_report_files`, `build_report_archive_contents` group and `build_report_files_view` are created when the first BuildReport object is analyzed; the `build_report_packed_*` tables and views are created when the first PackedAssets object is analyzed (a build report with no PackedAssets objects, such as some scene-only builds, will not have them). + ### Schema Overview Views automatically identify which build report each row belongs to, simplifying multi-report queries. To create custom queries, understand the table relationships: diff --git a/Documentation/command-analyze.md b/Documentation/command-analyze.md index 6cbfbf7..664764a 100644 --- a/Documentation/command-analyze.md +++ b/Documentation/command-analyze.md @@ -64,12 +64,12 @@ command works with the following types of input: |------------|-------------| | **AssetBundle build output** | The output path of an AssetBundle build | | **Addressables folder** | `StreamingAssets/aa` folder from a Player build, including BuildLayout files | -| **Entities content** | `StreamingAssets/ContentArchives` folder for [Entities](https://docs.unity3d.com/Packages/com.unity.entities@1.4/manual/content-management-intro.html) projects | | **Player Data folder** | The `Data` folder of a Unity Player build | | **Compressed Player builds** | The `data.unity3d` file will be analyzed like AssetBundles | | **ContentDirectory build output** | The output of [`BuildPipeline.BuildContentDirectory`](https://docs.unity3d.com/6000.6/Documentation/ScriptReference/BuildPipeline.BuildContentDirectory.html) (Unity 6.6+), ideally together with its build history folder. See [ContentDirectory builds](#contentdirectory-builds) | | **ContentLayout.json** | The layout file of a ContentDirectory build, imported into dedicated tables. Also useful on its own for querying a large layout with SQL. See [ContentLayout in the Analyze Database](contentlayout-database.md) | -| **BuildReport files** | The build report is typically found at a path like `Library/LastBuild.buildreport`and is a binary serialized file | +| **BuildReport files** | The build report is typically found inside `Library/BuildHistory` or at `Library/LastBuild.buildreport`. It is a binary serialized file | +| **Entities content** | `StreamingAssets/ContentArchives` folder for [Entities](https://docs.unity3d.com/Packages/com.unity.entities@1.4/manual/content-management-intro.html) projects | | **AssetDatabase Artifacts** | The tool will work to some extent with serialized files created in the AssetDatabase artifact storage, inside the Library folder | > **Note**: Some platforms require extracting content from platform-specific containers first (e.g., `.apk` files on Android). @@ -193,6 +193,7 @@ When `--skip-references` is used, some functionality is lost: * the `find-refs` command will not work * `view_material_shader_refs` and `view_material_texture_refs` will be empty * `script_object_view` will be empty +* `dangling_refs` will be empty * Queries that look at the relationship between objects will not work. For example the refs table is required to link between a `MonoBehaviour` and its `MonoScript`. When `--skip-crc` is used, the `objects.crc32` column will be 0 for all objects. This means: diff --git a/Documentation/contentdirectory-format.md b/Documentation/contentdirectory-format.md index f4192fa..1a73070 100644 --- a/Documentation/contentdirectory-format.md +++ b/Documentation/contentdirectory-format.md @@ -62,9 +62,8 @@ file — this is how the pipeline de-duplicates shared content automatically. Th clip per file, each named by its content hash. A Unity object references data inside a `.resS`/`.resource` file by a content-addressable path of the -form `cah:/` (a single slash; `cah` stands for "content-addressable hash"). For example an -AudioClip's `m_Source` is a string like `cah:/4226b5c16a50dab6eff0f08dd1253d4b`, which resolves to the -`.resource` file of that hash. +form `cah:/`. This, and the way it differs from AssetBundle and Player builds, is covered in +[References to resource files](#references-to-resource-files) below. The extensions are informational. The loading system identifies content by its hash (through the `cah:/` scheme), not by file extension, so the extension is not required to resolve content. Extensions @@ -303,8 +302,102 @@ flowchart TD > instead of the manifest dependency list. The GUIDs in the external table are also used for binary > SerializedFiles in the Editor (for example in AssetDatabase artifacts). +## References to resource files + +The bulk of a texture, mesh, audio clip, or video clip is not stored in the Content File itself but in a +companion data file: a `.resS` file for texture and mesh data, a `.resource` file for audio and video. +A Unity object points at its data with a content-addressable path of the form `cah:/` (a single +slash; `cah` stands for "content-addressable hash"). Unlike the `.cfid` placeholders used for +[references between Content Files](#references-between-content-files), the `` here is the real +content hash of the data file, so it resolves directly to `.resS` or `.resource` with no +manifest lookup. + +The reference build in `TestCommon/Data/LeadingEdgeBuilds/ContentDirectory` deliberately includes both +audio clips and a texture, so it exercises both kinds of data file. Dumping the Content File built from +`Assets/Audio/a.mp3` shows the AudioClip pointing at its `.resource` file, and dumping the one holding +`Assets/Textures/GreenStatic.png`'s texture data shows the Texture2D pointing at its `.resS` file: + +``` +ID: -6933612096100796476 (ClassID: 83) AudioClip + m_Name (string) a + ... + m_Resource (StreamedResource) + m_Source (string) cah:/4226b5c16a50dab6eff0f08dd1253d4b + m_Offset (FileSize) 0 + m_Size (UInt64) 47424 +``` +``` +ID: 1183010003894172340 (ClassID: 28) Texture2D + m_Name (string) GreenStatic + m_StreamData (StreamingInfo) + offset (UInt64) 0 + path (string) cah:/f0a44ad4a4babd121543fd44032928e7 +``` + +So the AudioClip's data lives in `4226b5c16a50dab6eff0f08dd1253d4b.resource` and the texture's in +`f0a44ad4a4babd121543fd44032928e7.resS`. + +### One resource per file, deduplicated across the build + +Each `.resS` and `.resource` file holds a **single** distinct texture, mesh, audio clip, or video clip. +Because the file is named by its content hash, if several objects reference identical binary data — even +across separate source assets — they all point at the same `cah:/`, and the build stores that data +only once. Deduplication of shared resource data is therefore automatic, the same way it is for Content +Files. + +This is the opposite of how AssetBundle and Player builds lay out resource data. There, all the `.resS` +data needed by a given SerializedFile is concatenated into one companion file belonging to that +SerializedFile (likewise for `.resource`), an object points into it by byte offset and size, and there is +no sharing between SerializedFiles — the same texture used by two bundles is written into each. Content +directories flip this to fine-grained, content-addressed, per-resource files that are shared across the +whole build. + +### How the relationship is recorded + +The link from a Content File to the resource files it uses is recorded in the `BinaryArtifacts` section +of [`ContentLayout.json`](contentlayout.md), not inside the Content File. Every artifact — each Content +File, each resource file, and the manifest — is an entry with a `Category` and a `Size`, and a Content +File's `ArtifactReferences` point at the resource artifacts it needs. Continuing the `a.mp3` example, its +`contentfile` artifact references the `audio` artifact whose content hash is the `cah:/` target seen +above: + +```json +{ "Index": 11, "ContentHash": "730d2d641a53eeb1e633f2ff60d730e9", "Category": "contentfile", "Size": 1288, "ArtifactReferences": [12] }, +{ "Index": 12, "ContentHash": "4226b5c16a50dab6eff0f08dd1253d4b", "Category": "audio", "Size": 47424 } +``` + +When [`analyze`](command-analyze.md) imports the layout it preserves this: the artifacts land in +`content_layout_binary_artifacts` (keyed by `artifact_index`, with `category` and `size`) and the +content-file → resource-file edges in `content_layout_artifact_references`. Resource files never appear +in the core `objects`/`serialized_files` tables, so `content_layout_binary_artifacts` is where their +sizes are recorded. The `content_layout_resource_files_view` walks that edge for you; joined with the +source-asset table it shows which resource file each asset was built into: + +```sql +SELECT sa.asset_path, rfv.data_filename, rfv.category, rfv.size +FROM content_layout_resource_files_view rfv +JOIN content_layout_source_assets sa ON sa.serialized_file_index = rfv.file_index +ORDER BY rfv.category, rfv.size; +``` +``` +asset_path data_filename category size +------------------------------- ----------------------------------------- -------- ------ +Assets/Audio/6.mp3 68c9d0b12420e1951eec7790bd0754fe.resource audio 30688 +Assets/Audio/a.mp3 4226b5c16a50dab6eff0f08dd1253d4b.resource audio 47424 +Assets/Textures/GreenStatic.png f0a44ad4a4babd121543fd44032928e7.resS texture 215970 +Resources/unity_builtin_extra ea7c9fa2d58ef71544726747da8ade6d.resS texture 524280 +``` + +The fourth row is the default sprite texture pulled in from `Resources/unity_builtin_extra` (see the +[granularity rules](#build-layout-granularity)). See [ContentLayout in the Analyze +Database](contentlayout-database.md) for the full set of `content_layout` tables and views. + ## Inspecting content directory output with UnityDataTool +The `dump` and `serialized-file` commands can be used to inspect serialized files in a content directory. +If the content is distributed inside archive files (e.g. `content0.archive`) then the `archive` command +can be used to view or extract the content. + When you run [`analyze`](command-analyze.md) on a content directory build, analyze the **build output folder and its matching build history folder together**, in a single `analyze` call, by passing both paths: @@ -319,15 +412,18 @@ references between Content Files live in the manifest rather than in the files t in the `dangling_refs` table and analyze prints a warning. Including the build history folder fixes both: the `ContentLayout.json` it contains provides the source-asset mapping (imported as the `content_layout` tables, see [ContentLayout in the Analyze Database](contentlayout-database.md)) and -the dependency information that analyze uses to resolve the references between Content Files. The -build report adds the per-object source mapping (the PackedAssets data, see -[BuildReport Support](buildreport.md)). +the dependency information that analyze uses to resolve the references between Content Files. Analyze verifies that the `ContentLayout.json` matches the build through the build's `BuildManifestHash.txt`, so a stale layout from a different build is rejected rather than producing misleading results. See the [`analyze` command](command-analyze.md#contentdirectory-builds) page for the exact input combinations. +Note: Passing in the entire build report folder in the build history means that the .buildreport file +will also be analyzed. That brings in statistics about the build (how long it took etc). For +content directory builds there is no PackedAssets data, but other build_report_* tables will be populated. +Call `analyze` with the path of the correct ContentLayout.json file, instead of the entire build report folder, if you do not need this extra data. + ## Related documentation | Topic | Description | diff --git a/Documentation/contentlayout-database.md b/Documentation/contentlayout-database.md index 606b9fe..5a8e8ac 100644 --- a/Documentation/contentlayout-database.md +++ b/Documentation/contentlayout-database.md @@ -65,7 +65,7 @@ Direct references between artifacts (`artifact_index` → `referenced_artifact_i | `content_layout_serialized_file_dependencies_view` | The dependency edges with filenames resolved on both sides. | | `content_layout_loadable_objects_view` | The loadables resolved to their analyzed object (`object`, `type`, `name`, `size`). | | `content_layout_binary_artifacts_view` | The artifacts with their derived on-disk filename (content hash + category-based extension). | -| `content_layout_data_files_view` | The `.resS`/`.resource` data files each serialized file uses, derived from the artifact graph. | +| `content_layout_resource_files_view` | The `.resS`/`.resource` data files each serialized file uses, derived from the artifact graph. Note: the same resource can be referenced by multiple serialized files. | ```sql -- which files was this source asset built into? diff --git a/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs b/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs index 912f444..31315d4 100644 --- a/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs +++ b/UnityDataTool.Tests/AnalyzeContentLayoutTests.cs @@ -136,9 +136,9 @@ SELECT dependency_index FROM content_layout_serialized_file_dependencies SQLTestHelper.AssertViewExists(db, "content_layout_serialized_file_dependencies_view"); SQLTestHelper.AssertViewExists(db, "content_layout_loadable_objects_view"); SQLTestHelper.AssertViewExists(db, "content_layout_binary_artifacts_view"); - SQLTestHelper.AssertViewExists(db, "content_layout_data_files_view"); + SQLTestHelper.AssertViewExists(db, "content_layout_resource_files_view"); - SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_data_files_view", 4, + SQLTestHelper.AssertQueryInt(db, "SELECT COUNT(*) FROM content_layout_resource_files_view", 4, "the reference build has 2 .resS and 2 .resource data files"); SQLTestHelper.AssertQueryString(db, "SELECT filename FROM content_layout_serialized_files_view WHERE is_builtin = 1",