stfs.h

Go to the documentation of this file.

 
#pragma once
 
#include <array>
#include <filesystem>
#include <span>
#include <string>
#include <string_view>
#include <memory>
#include <optional>
#include <utility>
#include <vector>
 
#include <rex/platform.h>
#include <rex/assert.h>
#include <rex/math.h>
#include <rex/memory/endian.h>
#include <rex/string/text.h>
#include <rex/result.h>
#include <rex/filesystem/xex.h>
#include <rex/kernel/types/xex.h>
#include <rex/kernel/types/content.h>
#include <rex/types.h>
#include <rex/filesystem/file_stat.h>
#include <rex/utility.h>
 
namespace rex::filesystem {

enum class XContentPackageType : u32 {
  kCon = 0x434F4E20,  
  kPirs = 0x50495253, 
  kLive = 0x4C495645, 
};
 
// NOLINTNEXTLINE(performance-enum-size): the on disk XContent header stores 4 bytes
enum class XContentVolumeType : u32 {
  kSTFS = 0, 
  kSVOD = 1, 
};
 
inline constexpr u32 kSTFSBlockSize = 0x1000;
 
inline constexpr u32 kSTFSMaxBlocks = 4913000;

inline constexpr std::array<u32, 3> kSTFSBlocksPerHashLevel = {170, 28900, kSTFSMaxBlocks};
 
#pragma pack(push, 1)

struct STFSVolumeDescriptor {
  u8 descriptor_length; 
  u8 version;
  union {
    u8 as_byte;
    struct {
      u8 read_only_format : 1;             
      u8 root_active_index : 1;            
      u8 directory_overallocated : 1;      
      u8 directory_index_bounds_valid : 1; 
    } bits;
  } flags;
  u16 file_table_block_count;        
  u8 file_table_block_number_raw[3]; 
  u8 top_hash_table_hash[0x14];      
  be_u32 total_block_count;
  be_u32 free_block_count;
 
  [[nodiscard]] constexpr u32 GetFileTableBlockNumber() const {
    return memory::LoadUint24LE(file_table_block_number_raw);
  }
 
  constexpr void SetFileTableBlockNumber(u32 value) {
    memory::StoreUint24LE(file_table_block_number_raw, value);
  }
 
  [[nodiscard]] constexpr bool IsValid() const {
    return descriptor_length == sizeof(STFSVolumeDescriptor);
  }
};
REX_STATIC_ASSERT_SIZE(STFSVolumeDescriptor, 0x24);
 
#pragma pack(pop)
 
enum class STFSHashState : u8 {
  kFree = 0,        
  kFreePresent = 1, 
  kInUse = 2,
};

struct STFSHashEntry {
  u8 sha1[0x14];
  be_u32 info_raw; 

  [[nodiscard]] u32 GetLevel0NextBlock() const { return info_raw & 0xFFFFFF; }
 
  void SetLevel0NextBlock(u32 value) { info_raw = (info_raw & ~0xFFFFFFu) | (value & 0xFFFFFF); }
 
  [[nodiscard]] STFSHashState GetLevel0AllocationState() const {
    return static_cast<STFSHashState>(static_cast<u8>((info_raw >> 30) & 0x3));
  }
 
  void SetLevel0AllocationState(STFSHashState value) {
    info_raw = (info_raw & ~0xC0000000u) | (static_cast<u32>(value) << 30);
  }
 
  [[nodiscard]] u32 GetLevelNFreeBlockCount() const { return info_raw & 0x7FFF; }
 
  void SetLevelNNumBlocksFree(u32 value) { info_raw = (info_raw & ~0x7FFFu) | (value & 0x7FFF); }
 
  [[nodiscard]] u32 GetLevelNNumBlocksUnk() const { return (info_raw >> 15) & 0x7FFF; }
 
  void SetLevelNNumBlocksUnk(u32 value) {
    info_raw = (info_raw & ~0x3FFF8000u) | ((value & 0x7FFF) << 15);
  }

  [[nodiscard]] bool IsLevelNActiveIndexSet() const { return (info_raw & 0x40000000) != 0; }
 
  void SetLevelNActiveIndex(bool value) {
    info_raw = (info_raw & ~0x40000000u) | (value ? 0x40000000u : 0u);
  }
 
  [[nodiscard]] bool IsLevelNWritable() const { return (info_raw & 0x80000000) != 0; }
 
  void SetLevelNWriteable(bool value) {
    info_raw = (info_raw & ~0x80000000u) | (value ? 0x80000000u : 0u);
  }
};
REX_STATIC_ASSERT_SIZE(STFSHashEntry, 0x18);
 
struct STFSHashTable {
  STFSHashEntry entries[170]; 
  be_u32 block_count;         
  u8 padding[12];
};
REX_STATIC_ASSERT_SIZE(STFSHashTable, 0x1000);
 
struct STFSDirectoryEntry {
  char name[40]; 
  struct {
    u8 name_length : 6;
    u8 contiguous : 1;
    u8 directory : 1;
  } flags;                         
  u8 valid_data_blocks_raw[3];     
  u8 allocated_data_blocks_raw[3]; 
  u8 start_block_number_raw[3];    
  be_u16 directory_index;          
  be_u32 length;
  be_u16 create_date;   
  be_u16 create_time;   
  be_u16 modified_date; 
  be_u16 modified_time; 
 
  [[nodiscard]] constexpr u32 GetValidDataBlocks() const {
    return memory::LoadUint24LE(valid_data_blocks_raw);
  }
 
  constexpr void SetValidDataBlocks(u32 value) {
    memory::StoreUint24LE(valid_data_blocks_raw, value);
  }
 
  [[nodiscard]] constexpr u32 GetAllocatedDataBlocks() const {
    return memory::LoadUint24LE(allocated_data_blocks_raw);
  }
 
  constexpr void SetAllocatedDataBlocks(u32 value) {
    memory::StoreUint24LE(allocated_data_blocks_raw, value);
  }
 
  [[nodiscard]] constexpr u32 GetStartBlockNumber() const {
    return memory::LoadUint24LE(start_block_number_raw);
  }
 
  constexpr void SetStartBlockNumber(u32 value) {
    memory::StoreUint24LE(start_block_number_raw, value);
  }
};
REX_STATIC_ASSERT_SIZE(STFSDirectoryEntry, 0x40);

struct STFSDirectoryBlock {
  STFSDirectoryEntry entries[0x40]; 
};
REX_STATIC_ASSERT_SIZE(STFSDirectoryBlock, 0x1000);

struct SVODDeviceDescriptor {
  u8 descriptor_length;
  u8 block_cache_element_count; 
  u8 worker_thread_processor;   
  u8 worker_thread_priority;
  u8 first_fragment_hash_entry[0x14]; 
  union {
    u8 as_byte;
    struct {
      u8 must_be_zero_for_future_usage : 6;
      u8 enhanced_gdf_layout : 1; 
      u8 zero_for_downlevel_clients : 1;
    } bits;
  } features;
  u8 data_block_count_raw[3]; 
  u8 start_data_block_raw[3]; 
  u8 reserved[5];
 
  [[nodiscard]] constexpr u32 GetDataBlockCount() const {
    return memory::LoadUint24LE(data_block_count_raw);
  }
 
  [[nodiscard]] constexpr u32 GetStartDataBlock() const {
    return memory::LoadUint24LE(start_data_block_raw);
  }
};
REX_STATIC_ASSERT_SIZE(SVODDeviceDescriptor, 0x24);
 
struct XContentLicense {
  be_u64 licensee_id; 
  be_u32 license_bits;
  be_u32 license_flags; 
};
REX_STATIC_ASSERT_SIZE(XContentLicense, 0x10);

struct XContentMedia {
  u8 series_id[0x10];
  u8 season_id[0x10];
  be_u16 season_number;
  be_u16 episode_number;
};
REX_STATIC_ASSERT_SIZE(XContentMedia, 0x24);

struct XContentAvatarAsset {
  be_u32 sub_category;
  be_u32 colorizable; 
  u8 asset_id[0x10];
  u8 skeleton_version_mask; 
  u8 reserved[0xB];
};
REX_STATIC_ASSERT_SIZE(XContentAvatarAsset, 0x24);
 
struct XContentAttributes {
  u8 profile_transfer : 1;
  u8 device_transfer : 1;
  u8 move_only_transfer : 1; 
  u8 kinect_enabled : 1;
  u8 disable_network_storage : 1;
  u8 deep_link_supported : 1;
  u8 reserved : 2;
};
REX_STATIC_ASSERT_SIZE(XContentAttributes, 1);
 
#pragma pack(push, 1)

struct XContentMetadata {
  static constexpr u32 kThumbLengthV1 = 0x4000;
  static constexpr u32 kThumbLengthV2 = 0x3D00;
  static constexpr u32 kNumLanguagesV1 = 9;
  static constexpr u32 kNumLanguagesV2 = 12;
 
  BE<kernel::XContentType> content_type;
  be_u32 metadata_version; 
  be_u64 content_size;
  kernel::XEXOptExecutionInfo execution_info; 
  u8 console_id[5];
  be_u64 profile_id;
  union {
    STFSVolumeDescriptor stfs;
    SVODDeviceDescriptor svod;
  } volume_descriptor;    
  be_u32 data_file_count; 
  be_u64 data_file_size;
  BE<XContentVolumeType> volume_type;
  be_u64 online_creator; 
  be_u32 category;
  u8 reserved2[0x20];
  union {
    XContentMedia media_data;              
    XContentAvatarAsset avatar_asset_data; 
  } metadata_v2;                           
  u8 device_id[0x14];
  union {
    be_u16 uint[kNumLanguagesV1][128];    
    char16_t chars[kNumLanguagesV1][128]; 
  } display_name_raw;                     
  union {
    be_u16 uint[kNumLanguagesV1][128];    
    char16_t chars[kNumLanguagesV1][128]; 
  } description_raw;                      
  union {
    be_u16 uint[64];    
    char16_t chars[64]; 
  } publisher_raw;
  union {
    be_u16 uint[64];    
    char16_t chars[64]; 
  } title_name_raw;
  union {
    u8 as_byte;
    XContentAttributes bits;
  } flags;
  be_u32 thumbnail_size;        
  be_u32 title_thumbnail_size;  
  u8 thumbnail[kThumbLengthV2]; 
  union {
    be_u16 uint[kNumLanguagesV2 - kNumLanguagesV1][128];    
    char16_t chars[kNumLanguagesV2 - kNumLanguagesV1][128]; 
  } display_name_ex_raw;              
  u8 title_thumbnail[kThumbLengthV2]; 
  union {
    be_u16 uint[kNumLanguagesV2 - kNumLanguagesV1][128];    
    char16_t chars[kNumLanguagesV2 - kNumLanguagesV1][128]; 
  } description_ex_raw; 

  [[nodiscard]] std::u16string GetDisplayName(kernel::XLanguage language) const;

  [[nodiscard]] std::u16string GetDescription(kernel::XLanguage language) const;
 
  [[nodiscard]] std::u16string GetPublisher() const;
 
  [[nodiscard]] std::u16string GetTitleName() const;

  bool SetDisplayName(kernel::XLanguage language, std::u16string_view value);

  bool SetDescription(kernel::XLanguage language, std::u16string_view value);

  void SetPublisher(std::u16string_view value);

  void SetTitleName(std::u16string_view value);
};
REX_STATIC_ASSERT_SIZE(XContentMetadata, 0x93D6);
 
namespace detail {
 
inline std::u16string ReadBigEndianUnits(const be_u16* units, size_t count) {
  std::u16string result;
  for (size_t i = 0; i < count && units[i] != 0; ++i) {
    result.push_back(static_cast<char16_t>(static_cast<u16>(units[i])));
  }
  return result;
}
 
inline std::optional<u32> FindLanguageSlot(kernel::XLanguage language) {
  if (language == kernel::XLanguage::kSChinese) {
    language = kernel::XLanguage::kSChineseDeprecated;
  }
  const u32 slot = static_cast<u32>(language) - 1;
  if (language == kernel::XLanguage::kInvalid || slot >= XContentMetadata::kNumLanguagesV2) {
    return std::nullopt;
  }
  return slot;
}
 
inline u32 LanguageSlot(kernel::XLanguage language) {
  return FindLanguageSlot(language).value_or(std::to_underlying(kernel::XLanguage::kEnglish) - 1);
}
 
}  // namespace detail
 
inline std::u16string XContentMetadata::GetDisplayName(kernel::XLanguage language) const {
  const u32 slot = detail::LanguageSlot(language);
  if (slot < kNumLanguagesV1) {
    return detail::ReadBigEndianUnits(display_name_raw.uint[slot], 128);
  }
  if (metadata_version >= 2) {
    return detail::ReadBigEndianUnits(display_name_ex_raw.uint[slot - kNumLanguagesV1], 128);
  }
  return {};
}
 
inline std::u16string XContentMetadata::GetDescription(kernel::XLanguage language) const {
  const u32 slot = detail::LanguageSlot(language);
  if (slot < kNumLanguagesV1) {
    return detail::ReadBigEndianUnits(description_raw.uint[slot], 128);
  }
  if (metadata_version >= 2) {
    return detail::ReadBigEndianUnits(description_ex_raw.uint[slot - kNumLanguagesV1], 128);
  }
  return {};
}
 
inline std::u16string XContentMetadata::GetPublisher() const {
  return detail::ReadBigEndianUnits(publisher_raw.uint, 64);
}
 
inline std::u16string XContentMetadata::GetTitleName() const {
  return detail::ReadBigEndianUnits(title_name_raw.uint, 64);
}
 
inline bool XContentMetadata::SetDisplayName(kernel::XLanguage language,
                                             std::u16string_view value) {
  const std::optional<u32> found = detail::FindLanguageSlot(language);
  if (!found) {
    return false;
  }
  const u32 slot = *found;
  char16_t* destination = nullptr;
  if (slot < kNumLanguagesV1) {
    destination = display_name_raw.chars[slot];
  } else if (metadata_version >= 2) {
    destination = display_name_ex_raw.chars[slot - kNumLanguagesV1];
  }
  if (!destination) {
    return false;
  }
  string::CopyAndSwapTruncating(destination, value, CountOf(display_name_raw.chars[0]));
  return true;
}
 
inline bool XContentMetadata::SetDescription(kernel::XLanguage language,
                                             std::u16string_view value) {
  const std::optional<u32> found = detail::FindLanguageSlot(language);
  if (!found) {
    return false;
  }
  const u32 slot = *found;
  char16_t* destination = nullptr;
  if (slot < kNumLanguagesV1) {
    destination = description_raw.chars[slot];
  } else if (metadata_version >= 2) {
    destination = description_ex_raw.chars[slot - kNumLanguagesV1];
  }
  if (!destination) {
    return false;
  }
  string::CopyAndSwapTruncating(destination, value, CountOf(description_raw.chars[0]));
  return true;
}
 
inline void XContentMetadata::SetPublisher(std::u16string_view value) {
  string::CopyAndSwapTruncating(publisher_raw.chars, value, CountOf(publisher_raw.chars));
}
 
inline void XContentMetadata::SetTitleName(std::u16string_view value) {
  string::CopyAndSwapTruncating(title_name_raw.chars, value, CountOf(title_name_raw.chars));
}
 
struct XContentHeader {
  BE<XContentPackageType> magic;
  u8 signature[0x228];
  XContentLicense licenses[0x10];
  u8 content_id[0x14]; 
  be_u32 header_size;  

  [[nodiscard]] bool IsMagicValid() const {
    const XContentPackageType type = memory::load<BE<XContentPackageType>>(&magic);
    return type == XContentPackageType::kCon || type == XContentPackageType::kLive ||
           type == XContentPackageType::kPirs;
  }
};
REX_STATIC_ASSERT_SIZE(XContentHeader, 0x344);
 
#pragma pack(pop)

struct XContentPackageHeader {
  XContentHeader header;
  XContentMetadata metadata;
};
REX_STATIC_ASSERT_SIZE(XContentPackageHeader, 0x971A);
 
}  // namespace rex::filesystem
 
namespace rex::filesystem {
 
enum class ContainerType : u8 {
  kSTFS, 
  kSVOD, 
  kGDFX,
};

struct ContainerExtent {
  u32 file = 0;
  u64 offset = 0;
  u64 length = 0;
};
 
struct ContainerNode {
  FileStat info;
  u32 parent = 0;                       
  std::vector<u32> children;            
  std::vector<ContainerExtent> extents; 
};

class IContainerImage {
 public:
  virtual ~IContainerImage() = default;
 
  [[nodiscard]] virtual ContainerType GetType() const = 0;

  [[nodiscard]] virtual const XContentPackageHeader* GetHeader() const = 0;
 
  [[nodiscard]] virtual const std::filesystem::path& GetPath() const = 0;

  [[nodiscard]] virtual u64 GetTotalSize() const = 0;

  [[nodiscard]] virtual std::span<const ContainerNode> GetNodes() const = 0;
 
  [[nodiscard]] virtual const ContainerNode& GetRoot() const = 0;

  [[nodiscard]] virtual const ContainerNode* Find(std::string_view path) const = 0;

  [[nodiscard]] virtual std::string GetNodePath(const ContainerNode& node) const = 0;

  [[nodiscard]] virtual Result<size_t> Read(const ContainerNode& node, u64 offset,
                                            std::span<u8> buffer) const = 0;

  [[nodiscard]] virtual std::span<const u8> View(const ContainerNode& node) const = 0;

  [[nodiscard]] virtual Result<void> Extract(const std::filesystem::path& destination) const = 0;
};
 
}  // namespace rex::filesystem