Update CacheSystem

修复原生文件每次获取都重复拷贝的问题
pull/51/head
hevinci 2022-10-18 10:19:17 +08:00
parent 9e2efe3717
commit c449a070b4
9 changed files with 349 additions and 282 deletions

View File

@ -0,0 +1,181 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
namespace YooAsset
{
/// <summary>
/// 本地缓存文件验证器
/// </summary>
internal abstract class CacheVerifier
{
public abstract void InitVerifier(List<VerifyInfo> verifyInfos);
public abstract bool UpdateVerifier();
public abstract float GetVerifierProgress();
public List<VerifyInfo> VerifySuccessList { protected set; get; }
public List<VerifyInfo> VerifyFailList { protected set; get; }
}
/// <summary>
/// 本地缓存文件验证器(线程版)
/// </summary>
internal class CacheVerifierWithThread : CacheVerifier
{
private readonly ThreadSyncContext _syncContext = new ThreadSyncContext();
private readonly List<VerifyInfo> _waitingList = new List<VerifyInfo>(1000);
private readonly List<VerifyInfo> _verifyingList = new List<VerifyInfo>(100);
private int _verifyMaxNum;
private int _verifyTotalCount;
public override void InitVerifier(List<VerifyInfo> verifyInfos)
{
_waitingList.AddRange(verifyInfos);
VerifySuccessList = new List<VerifyInfo>(verifyInfos.Count);
VerifyFailList = new List<VerifyInfo>(verifyInfos.Count);
// 设置同时验证的最大数
ThreadPool.GetMaxThreads(out int workerThreads, out int ioThreads);
YooLogger.Log($"Work threads : {workerThreads}, IO threads : {ioThreads}");
_verifyMaxNum = Math.Min(workerThreads, ioThreads);
_verifyTotalCount = _waitingList.Count;
if (_verifyMaxNum < 1)
_verifyMaxNum = 1;
}
public override bool UpdateVerifier()
{
_syncContext.Update();
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
return true;
if (_verifyingList.Count >= _verifyMaxNum)
return false;
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (_verifyingList.Count >= _verifyMaxNum)
break;
var verifyIno = _waitingList[i];
if (VerifyFileWithThread(verifyIno))
{
_waitingList.RemoveAt(i);
_verifyingList.Add(verifyIno);
}
else
{
YooLogger.Warning("The thread pool is failed queued.");
break;
}
}
return false;
}
public override float GetVerifierProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(VerifySuccessList.Count + VerifyFailList.Count) / _verifyTotalCount;
}
private bool VerifyFileWithThread(VerifyInfo verifyInfo)
{
return ThreadPool.QueueUserWorkItem(new WaitCallback(VerifyInThread), verifyInfo);
}
private void VerifyInThread(object infoObj)
{
VerifyInfo verifyInfo = (VerifyInfo)infoObj;
verifyInfo.Result = CacheSystem.VerifyBundle(verifyInfo.VerifyBundle, CacheSystem.InitVerifyLevel);
_syncContext.Post(VerifyCallback, verifyInfo);
}
private void VerifyCallback(object obj)
{
VerifyInfo verifyIno = (VerifyInfo)obj;
if (verifyIno.Result == EVerifyResult.Succeed)
{
VerifySuccessList.Add(verifyIno);
CacheSystem.CacheBundle(verifyIno.VerifyBundle);
}
else
{
VerifyFailList.Add(verifyIno);
// NOTE不期望删除断点续传的资源文件
/*
if (File.Exists(patchBundle.CachedBundleFilePath))
File.Delete(patchBundle.CachedBundleFilePath);
*/
}
_verifyingList.Remove(verifyIno);
}
}
/// <summary>
/// 本地缓存文件验证器(非线程版)
/// </summary>
internal class CacheVerifierWithoutThread : CacheVerifier
{
private readonly List<VerifyInfo> _waitingList = new List<VerifyInfo>(1000);
private readonly List<VerifyInfo> _verifyingList = new List<VerifyInfo>(100);
private int _verifyMaxNum;
private int _verifyTotalCount;
public override void InitVerifier(List<VerifyInfo> verifyInfos)
{
_waitingList.AddRange(verifyInfos);
VerifySuccessList = new List<VerifyInfo>(verifyInfos.Count);
VerifyFailList = new List<VerifyInfo>(verifyInfos.Count);
// 设置同时验证的最大数
_verifyMaxNum = 32;
_verifyTotalCount = _waitingList.Count;
}
public override bool UpdateVerifier()
{
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
return true;
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (_verifyingList.Count >= _verifyMaxNum)
break;
var verifyIno = _waitingList[i];
VerifyFileWithoutThread(verifyIno);
_waitingList.RemoveAt(i);
_verifyingList.Add(verifyIno);
}
_verifyingList.Clear();
return false;
}
public override float GetVerifierProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(VerifySuccessList.Count + VerifyFailList.Count) / _verifyTotalCount;
}
private void VerifyFileWithoutThread(VerifyInfo verifyIno)
{
var verifyResult = CacheSystem.VerifyAndCacheBundle(verifyIno.VerifyBundle, CacheSystem.InitVerifyLevel);
if (verifyResult == EVerifyResult.Succeed)
{
VerifySuccessList.Add(verifyIno);
}
else
{
VerifyFailList.Add(verifyIno);
// NOTE不期望删除断点续传的资源文件
/*
if (File.Exists(patchBundle.CachedBundleFilePath))
File.Delete(patchBundle.CachedBundleFilePath);
*/
}
}
}
}

View File

@ -1,247 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
namespace YooAsset
{
/// <summary>
/// 本地缓存文件验证器
/// </summary>
internal abstract class PatchCacheVerifier
{
public abstract bool InitVerifier(HostPlayModeImpl impl, bool weaklyUpdate);
public abstract bool UpdateVerifier();
public abstract float GetVerifierProgress();
public int VerifySuccessCount { protected set; get; } = 0;
public int VerifyFailCount { protected set; get; } = 0;
}
/// <summary>
/// 本地缓存文件验证器(线程版)
/// </summary>
internal class PatchCacheVerifierWithThread : PatchCacheVerifier
{
private class ThreadInfo
{
public EVerifyResult Result;
public string FilePath { private set; get; }
public PatchBundle Bundle { private set; get; }
public ThreadInfo(string filePath, PatchBundle bundle)
{
FilePath = filePath;
Bundle = bundle;
}
}
private readonly ThreadSyncContext _syncContext = new ThreadSyncContext();
private readonly List<PatchBundle> _waitingList = new List<PatchBundle>(1000);
private readonly List<PatchBundle> _verifyingList = new List<PatchBundle>(100);
private int _verifyMaxNum;
private int _verifyTotalCount;
public override bool InitVerifier(HostPlayModeImpl impl, bool weaklyUpdate)
{
// 遍历所有文件然后验证并缓存合法文件
foreach (var patchBundle in impl.LocalPatchManifest.BundleList)
{
// 忽略缓存文件
if (CacheSystem.IsCached(patchBundle))
continue;
// 忽略APP资源
if (impl.IsBuildinPatchBundle(patchBundle))
continue;
// 注意:在弱联网模式下,我们需要验证指定资源版本的所有资源完整性
if (weaklyUpdate)
{
string filePath = patchBundle.CachedFilePath;
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
else
return false;
}
else
{
string filePath = patchBundle.CachedFilePath;
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
}
}
// 设置同时验证的最大数
ThreadPool.GetMaxThreads(out int workerThreads, out int ioThreads);
YooLogger.Log($"Work threads : {workerThreads}, IO threads : {ioThreads}");
_verifyMaxNum = Math.Min(workerThreads, ioThreads);
_verifyTotalCount = _waitingList.Count;
if (_verifyMaxNum < 1)
_verifyMaxNum = 1;
return true;
}
public override bool UpdateVerifier()
{
_syncContext.Update();
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
return true;
if (_verifyingList.Count >= _verifyMaxNum)
return false;
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (_verifyingList.Count >= _verifyMaxNum)
break;
var patchBundle = _waitingList[i];
if (VerifyFile(patchBundle))
{
_waitingList.RemoveAt(i);
_verifyingList.Add(patchBundle);
}
else
{
YooLogger.Warning("The thread pool is failed queued.");
break;
}
}
return false;
}
public override float GetVerifierProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(VerifySuccessCount + VerifyFailCount) / _verifyTotalCount;
}
private bool VerifyFile(PatchBundle patchBundle)
{
string filePath = patchBundle.CachedFilePath;
ThreadInfo info = new ThreadInfo(filePath, patchBundle);
return ThreadPool.QueueUserWorkItem(new WaitCallback(VerifyInThread), info);
}
private void VerifyInThread(object infoObj)
{
ThreadInfo info = (ThreadInfo)infoObj;
info.Result = CacheSystem.VerifyBundle(info.Bundle, CacheSystem.InitVerifyLevel);
_syncContext.Post(VerifyCallback, info);
}
private void VerifyCallback(object obj)
{
ThreadInfo info = (ThreadInfo)obj;
if (info.Result == EVerifyResult.Succeed)
{
VerifySuccessCount++;
CacheSystem.CacheBundle(info.Bundle);
}
else
{
VerifyFailCount++;
YooLogger.Warning($"Failed to verify file : {info.Bundle.CachedFilePath}");
// NOTE不期望删除断点续传的资源文件
/*
if (File.Exists(patchBundle.CachedBundleFilePath))
File.Delete(patchBundle.CachedBundleFilePath);
*/
}
_verifyingList.Remove(info.Bundle);
}
}
/// <summary>
/// 本地缓存文件验证器(非线程版)
/// </summary>
internal class PatchCacheVerifierWithoutThread : PatchCacheVerifier
{
private readonly List<PatchBundle> _waitingList = new List<PatchBundle>(1000);
private readonly List<PatchBundle> _verifyingList = new List<PatchBundle>(100);
private int _verifyMaxNum;
private int _verifyTotalCount;
public override bool InitVerifier(HostPlayModeImpl impl, bool weaklyUpdate)
{
// 遍历所有文件然后验证并缓存合法文件
foreach (var patchBundle in impl.LocalPatchManifest.BundleList)
{
// 忽略缓存文件
if (CacheSystem.IsCached(patchBundle))
continue;
// 忽略APP资源
if (impl.IsBuildinPatchBundle(patchBundle))
continue;
// 注意:在弱联网模式下,我们需要验证指定资源版本的所有资源完整性
if (weaklyUpdate)
{
string filePath = patchBundle.CachedFilePath;
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
else
return false;
}
else
{
string filePath = patchBundle.CachedFilePath;
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
}
}
// 设置同时验证的最大数
_verifyMaxNum = 32;
_verifyTotalCount = _waitingList.Count;
return true;
}
public override bool UpdateVerifier()
{
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
return true;
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (_verifyingList.Count >= _verifyMaxNum)
break;
var patchBundle = _waitingList[i];
VerifyFile(patchBundle);
_waitingList.RemoveAt(i);
_verifyingList.Add(patchBundle);
}
_verifyingList.Clear();
return false;
}
public override float GetVerifierProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(VerifySuccessCount + VerifyFailCount) / _verifyTotalCount;
}
private void VerifyFile(PatchBundle patchBundle)
{
var verifyResult = CacheSystem.VerifyAndCacheBundle(patchBundle, CacheSystem.InitVerifyLevel);
if (verifyResult == EVerifyResult.Succeed)
{
VerifySuccessCount++;
}
else
{
VerifyFailCount++;
YooLogger.Warning($"Failed to verify file : {patchBundle.CachedFilePath}");
// NOTE不期望删除断点续传的资源文件
/*
if (File.Exists(patchBundle.CachedBundleFilePath))
File.Delete(patchBundle.CachedBundleFilePath);
*/
}
}
}
}

View File

@ -0,0 +1,33 @@

namespace YooAsset
{
internal class VerifyInfo
{
/// <summary>
/// 验证的资源文件是否为内置资源
/// </summary>
public bool IsBuildinFile { private set; get; }
/// <summary>
/// 验证的资源包实例
/// </summary>
public PatchBundle VerifyBundle { private set; get; }
/// <summary>
/// 验证的文件路径
/// </summary>
public string VerifyFilePath { private set; get; }
/// <summary>
/// 验证结果
/// </summary>
public EVerifyResult Result;
public VerifyInfo(bool isBuildinFile, PatchBundle verifyBundle)
{
IsBuildinFile = isBuildinFile;
VerifyBundle = verifyBundle;
VerifyFilePath = verifyBundle.CachedFilePath;
}
}
}

View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: a6296859f09655c4191594304ddf378f
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@ -67,29 +67,39 @@ namespace YooAsset
private enum ESteps
{
None,
Update,
LoadAppManifest,
InitVerifyingCache,
UpdateVerifyingCache,
Done,
}
private readonly OfflinePlayModeImpl _impl;
private AppManifestLoader _appManifestLoader;
private readonly AppManifestLoader _appManifestLoader;
private readonly CacheVerifier _patchCacheVerifier;
private ESteps _steps = ESteps.None;
private float _verifyTime;
internal OfflinePlayModeInitializationOperation(OfflinePlayModeImpl impl, string buildinPackageName)
{
_impl = impl;
_appManifestLoader = new AppManifestLoader(buildinPackageName);
#if UNITY_WEBGL
_patchCacheVerifier = new CacheVerifierWithoutThread();
#else
_patchCacheVerifier = new CacheVerifierWithThread();
#endif
}
internal override void Start()
{
_steps = ESteps.Update;
_steps = ESteps.LoadAppManifest;
}
internal override void Update()
{
if (_steps == ESteps.None || _steps == ESteps.Done)
return;
if (_steps == ESteps.Update)
if (_steps == ESteps.LoadAppManifest)
{
_appManifestLoader.Update();
Progress = _appManifestLoader.Progress();
@ -103,10 +113,29 @@ namespace YooAsset
Error = _appManifestLoader.Error;
}
else
{
_steps = ESteps.InitVerifyingCache;
_impl.SetAppPatchManifest(_appManifestLoader.Result);
}
}
if (_steps == ESteps.InitVerifyingCache)
{
var verifyInfos = _impl.GetVerifyInfoList();
_patchCacheVerifier.InitVerifier(verifyInfos);
_verifyTime = UnityEngine.Time.realtimeSinceStartup;
_steps = ESteps.UpdateVerifyingCache;
}
if (_steps == ESteps.UpdateVerifyingCache)
{
Progress = _patchCacheVerifier.GetVerifierProgress();
if (_patchCacheVerifier.UpdateVerifier())
{
_steps = ESteps.Done;
Status = EOperationStatus.Succeed;
_impl.SetAppPatchManifest(_appManifestLoader.Result);
float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyTime;
YooLogger.Log($"Verify result : Success {_patchCacheVerifier.VerifySuccessList.Count}, Fail {_patchCacheVerifier.VerifyFailList.Count}, Elapsed time {costTime} seconds");
}
}
}

View File

@ -65,9 +65,9 @@ namespace YooAsset
private readonly string _packageName;
private readonly string _packageCRC;
private readonly int _timeout;
private ESteps _steps = ESteps.None;
private UnityWebDataRequester _downloader;
private PatchCacheVerifier _patchCacheVerifier;
private CacheVerifier _cacheVerifier;
private ESteps _steps = ESteps.None;
private float _verifyTime;
internal HostPlayModeUpdateManifestOperation(HostPlayModeImpl impl, string packageName, string packageCRC, int timeout)
@ -78,9 +78,9 @@ namespace YooAsset
_timeout = timeout;
#if UNITY_WEBGL
_patchCacheVerifier = new PatchCacheVerifierWithoutThread();
_cacheVerifier = new CacheVerifierWithoutThread();
#else
_patchCacheVerifier = new PatchCacheVerifierWithThread();
_cacheVerifier = new CacheVerifierWithThread();
#endif
}
internal override void Start()
@ -154,20 +154,21 @@ namespace YooAsset
if (_steps == ESteps.InitVerifyingCache)
{
_patchCacheVerifier.InitVerifier(_impl, false);
var verifyInfos = _impl.GetVerifyInfoList(false);
_cacheVerifier.InitVerifier(verifyInfos);
_verifyTime = UnityEngine.Time.realtimeSinceStartup;
_steps = ESteps.UpdateVerifyingCache;
}
if (_steps == ESteps.UpdateVerifyingCache)
{
Progress = _patchCacheVerifier.GetVerifierProgress();
if (_patchCacheVerifier.UpdateVerifier())
Progress = _cacheVerifier.GetVerifierProgress();
if (_cacheVerifier.UpdateVerifier())
{
_steps = ESteps.Done;
Status = EOperationStatus.Succeed;
float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyTime;
YooLogger.Log($"Verify result : Success {_patchCacheVerifier.VerifySuccessCount}, Fail {_patchCacheVerifier.VerifyFailCount}, Elapsed time {costTime} seconds");
YooLogger.Log($"Verify result : Success {_cacheVerifier.VerifySuccessList.Count}, Fail {_cacheVerifier.VerifyFailList.Count}, Elapsed time {costTime} seconds");
}
}
}
@ -254,7 +255,7 @@ namespace YooAsset
private readonly string _packageName;
private readonly string _packageCRC;
private ESteps _steps = ESteps.None;
private PatchCacheVerifier _patchCacheVerifier;
private CacheVerifier _cacheVerifier;
private float _verifyTime;
internal HostPlayModeWeaklyUpdateManifestOperation(HostPlayModeImpl impl, string packageName, string packageCRC)
@ -264,9 +265,9 @@ namespace YooAsset
_packageCRC = packageCRC;
#if UNITY_WEBGL
_patchCacheVerifier = new PatchCacheVerifierWithoutThread();
_cacheVerifier = new CacheVerifierWithoutThread();
#else
_patchCacheVerifier = new PatchCacheVerifierWithThread();
_cacheVerifier = new CacheVerifierWithThread();
#endif
}
internal override void Start()
@ -286,36 +287,41 @@ namespace YooAsset
if (_steps == ESteps.InitVerifyingCache)
{
if (_patchCacheVerifier.InitVerifier(_impl, true))
{
_verifyTime = UnityEngine.Time.realtimeSinceStartup;
_steps = ESteps.UpdateVerifyingCache;
}
else
{
_steps = ESteps.Done;
Status = EOperationStatus.Failed;
Error = $"The package resource {_packageName}_{_packageCRC} content is not complete !";
}
var verifyInfos = _impl.GetVerifyInfoList(true);
_cacheVerifier.InitVerifier(verifyInfos);
_verifyTime = UnityEngine.Time.realtimeSinceStartup;
_steps = ESteps.UpdateVerifyingCache;
}
if (_steps == ESteps.UpdateVerifyingCache)
{
Progress = _patchCacheVerifier.GetVerifierProgress();
if (_patchCacheVerifier.UpdateVerifier())
Progress = _cacheVerifier.GetVerifierProgress();
if (_cacheVerifier.UpdateVerifier())
{
float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyTime;
YooLogger.Log($"Verify result : Success {_patchCacheVerifier.VerifySuccessCount}, Fail {_patchCacheVerifier.VerifyFailCount}, Elapsed time {costTime} seconds");
if (_patchCacheVerifier.VerifyFailCount > 0)
YooLogger.Log($"Verify result : Success {_cacheVerifier.VerifySuccessList.Count}, Fail {_cacheVerifier.VerifyFailList.Count}, Elapsed time {costTime} seconds");
bool verifySucceed = true;
foreach (var verifyInfo in _cacheVerifier.VerifyFailList)
{
// 注意:跳过内置资源文件
if (verifyInfo.IsBuildinFile)
continue;
verifySucceed = false;
YooLogger.Warning($"Failed verify file : {verifyInfo.VerifyFilePath}");
}
if (verifySucceed)
{
_steps = ESteps.Done;
Status = EOperationStatus.Failed;
Error = $"The package resource {_packageName}_{_packageCRC} content has verify failed file !";
Status = EOperationStatus.Succeed;
}
else
{
_steps = ESteps.Done;
Status = EOperationStatus.Succeed;
Status = EOperationStatus.Failed;
Error = $"The package resource {_packageName}_{_packageCRC} content has verify failed file !";
}
}
}

View File

@ -288,6 +288,38 @@ namespace YooAsset
return bundleInfo;
}
internal List<VerifyInfo> GetVerifyInfoList(bool weaklyUpdateMode)
{
List<VerifyInfo> result = new List<VerifyInfo>(LocalPatchManifest.BundleList.Count);
// 遍历所有文件然后验证并缓存合法文件
foreach (var patchBundle in LocalPatchManifest.BundleList)
{
// 忽略缓存文件
if (CacheSystem.IsCached(patchBundle))
continue;
// 注意:在弱联网模式下,我们需要验证指定资源版本的所有资源完整性
if (weaklyUpdateMode)
{
bool isBuildinFile = IsBuildinPatchBundle(patchBundle);
VerifyInfo verifyInfo = new VerifyInfo(isBuildinFile, patchBundle);
result.Add(verifyInfo);
}
else
{
string filePath = patchBundle.CachedFilePath;
if (File.Exists(filePath))
{
bool isBuildinFile = IsBuildinPatchBundle(patchBundle);
VerifyInfo verifyInfo = new VerifyInfo(isBuildinFile, patchBundle);
result.Add(verifyInfo);
}
}
}
return result;
}
internal void SetLocalPatchManifest(PatchManifest patchManifest)
{
LocalPatchManifest = patchManifest;

View File

@ -1,6 +1,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
namespace YooAsset
{
@ -30,7 +31,28 @@ namespace YooAsset
return _appPatchManifest.HumanReadableVersion;
}
// 设置资源清单
internal List<VerifyInfo> GetVerifyInfoList()
{
List<VerifyInfo> result = new List<VerifyInfo>(_appPatchManifest.BundleList.Count);
// 遍历所有文件然后验证并缓存合法文件
foreach (var patchBundle in _appPatchManifest.BundleList)
{
// 忽略缓存文件
if (CacheSystem.IsCached(patchBundle))
continue;
string filePath = patchBundle.CachedFilePath;
if (File.Exists(filePath))
{
bool isBuildinFile = true;
VerifyInfo verifyInfo = new VerifyInfo(isBuildinFile, patchBundle);
result.Add(verifyInfo);
}
}
return result;
}
internal void SetAppPatchManifest(PatchManifest patchManifest)
{
_appPatchManifest = patchManifest;