a 0Sic@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddl mZddl mZddl mZddl mZdd l mZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlm Z ddlm!Z!ddlm"Z"ddl#m$Z$ddl%m&Z&ddl%m'Z(ddl%m)Z)ddl%m*Z*ddl%m+Z+ddl,m-Z-ddl,m.Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl4m6Z6dd l4m7Z7dd!l8m9Z9dd"l8m:Z;dd#lm?Z?dd%l>m@Z@dd&l>mAZAdd'l>mBZBdd(l>mCZCdd)lDmEZEdaFdaGeHZId*ZJd+ZKdaLd,d-ZMeEd.gd/d0d1ZNd2d3ZOd4d5ZPGd6d7d7eQZRGd8d9d9eQZSGd:d;d;eQZTGdd?ZVdejWdd@fdAdBZXdCdDZYdEdFZZdGdHZ[eBj\dIdJZ]GdKdLdLeQZ^eEdMgd/dkdNdOZ_dPdQZ`GdRdSdSe^ZaGdTdUdUe^ZbdVZcGdWdXdXe^ZdGdYdZdZejeZfGd[d\d\eQZgd]d^Zhd_d`ZidadbZjdcddZkeEdegd/Gdfdgdge5jlZmeEdegd/Gdhdidie5jlZndS)lz/Utilities for saving/loading Trackable objects.N)trackable_object_graph_pb2)checkpoint_management)checkpoint_options)functional_saver) graph_view)restore)utilsession)context) def_function)executor) constant_op)dtypes) errors_implops) tensor_shape) tensor_util)file_io) array_ops) gen_io_ops)init_ops)variable_scope) variables)gfile) tf_logging) utils_impl)metrics) autotrackable)base)data_structures)py_checkpoint_reader)saver)saveable_object_util)compat) deprecation)object_identity) tf_contextlib) tf_inspect) tf_exportZ checkpoint_v1Z checkpoint_v2cCs||kr dSt||dS)Nri@B)round)start_time_secondsend_time_secondsr.c/var/www/html/django/DPS/env/lib/python3.9/site-packages/tensorflow/python/checkpoint/checkpoint.py_get_duration_microsecondsTsr0z/__internal__.tracking.register_session_provider)v1cCs|adSN)_SESSION_PROVIDER)Zsession_providerr.r.r/register_session_provider[sr4cCs"t}|durtdurt}|Sr2)rget_default_sessionr3r r.r.r/ get_sessionds r6cCs0d}td|}|D]}|t|7}q|S)z2Calculates filesize of checkpoint based on prefix.rz{}*)globformatrCalculateFileSize)prefixsizefilesfiler.r.r/_get_checkpoint_sizens r>c@s,eZdZdZddgZddZeddZdS) ObjectGraphProtoPrettyPrinteraLazily traverses an object graph proto to pretty print names. If no calls to `node_names` are made this object has no performance overhead. On the other hand, it will only traverse the object graph once, so repeated naming is cheap after the first. _object_graph_proto_node_name_cachecCs||_d|_dSr2)r@rA)selfobject_graph_protor.r.r/__init__sz&ObjectGraphProtoPrettyPrinter.__init__c Cs|jdur|jSi}d|d<tdg}|rz|}|jj|}|jD]0}|j|vrF|||jf||j<| |jqFq(i}| D]\}}d |||<qt |jjD]:\}}|j D]*}||d|jd||j||j<qq||_|S)z@Lazily creates a mapping from node id to ("path", "to", "root").N)z(root)r.z 's state 'z' for )rA collectionsdequepopleftr@nodeschildrennode_id local_nameappenditemsjoin enumerateslot_variables slot_nameoriginal_variable_node_idslot_variable_node_id) rBZ path_to_rootto_visitrKobjchild node_namesnodeslot_referencer.r.r/rXs0       z(ObjectGraphProtoPrettyPrinter.node_namesN)__name__ __module__ __qualname____doc__ __slots__rDpropertyrXr.r.r.r/r?ys r?c@s0eZdZdZgdZddZddZddZd S) $_CheckpointRestoreCoordinatorDeleterzDDeleter to avoid overriding _CheckpointRestoreCoordinator.__del__().expect_partialrCmatched_proto_idsunused_attributescCs||_||_||_||_dSr2rb)rBrcrCrdrer.r.r/rDsz-_CheckpointRestoreCoordinatorDeleter.__init__cCs ||_dSr2)rcrBrcr.r.r/set_expect_partialsz7_CheckpointRestoreCoordinatorDeleter.set_expect_partialc Cs|jr dStdurt}ntj}g}g}t|j}t|jjD]*\}}|jsLq<||j vr<| |j |q<|j D]\}}| |j ||fqr|s|r|d|D]}|d|q|D]\}} |d|d| qdS)NagDetecting that an object or model or tf.train.Checkpoint is being deleted with unrestored values. See the following logs for the specific values in question. To silence these warnings, use `status.expect_partial()`. See https://www.tensorflow.org/api_docs/python/tf/train/Checkpoint#restorefor details about the status object returned by the restore function.z?Value in checkpoint could not be found in the restored object: zRAn attribute in the restored object could not be found in the checkpoint. Object: z , attribute: )rcloggingprintwarningr?rCrPrI attributesrdrMrXrerN) rBZlog_fnZunused_nodes_in_checkpointZunrestored_attributes_in_objectpretty_printerrKrYattribute_name node_pathattrr.r.r/__del__s:     z,_CheckpointRestoreCoordinatorDeleter.__del__N)r[r\r]r^r_rDrgrpr.r.r.r/ras rac@sDeZdZdZddZeddZejddZddZd d d Z d S) _CheckpointRestoreCoordinatorz4Holds the status of an object-based checkpoint load.c Cs||_||_t|_i|_t|_t |_ t |_ ||_||_||_|jdur`t||_||_||_g|_||_||_d|_i|_i|_d|_t|jjD]:\} } | j D]*} |j!| j"g#t$j%| | j&| j'dqqt(|j|j|j |j|_)||_*dS)aSpecify the checkpoint being loaded. Args: object_graph_proto: The TrackableObjectGraph protocol buffer associated with this checkpoint. save_path: A string, the path to the checkpoint, as returned by `tf.train.latest_checkpoint`. save_path_tensor: A string `Tensor` which contains or will be fed the save path. reader: A `CheckpointReader` for `save_path`. If None, `_CheckpointRestoreCoordinator` will initialize one itself. restore_op_cache: A dictionary shared between `_CheckpointRestoreCoordinator`s for the same Python objects, used to look up restore ops by name to avoid re-creating them across multiple `restore()` calls. graph_view: A graph_view_lib.ObjectGraphView object for the restored objects. options: A CheckpointOptions object. saveables_cache: An optional cache storing previously created SaveableObjects created for each Trackable. Maps Trackables to a dictionary of attribute names to Trackable. NF) optimizer_idslot_variable_idrR)+optionsrCruid restore_uidreweakrefWeakValueDictionaryobject_by_proto_idsetrdr'ObjectIdentityWeakSetall_python_objectssave_path_tensorZsave_path_stringreaderr"NewCheckpointReaderget_variable_to_dtype_map dtype_mapget_variable_to_shape_mapZ shape_map restore_opsrestore_ops_by_namernew_restore_ops_callbackZdeferred_slot_restorationsslot_restorationsexpect_partial_attrrPrIrQ setdefaultrSrMr _SlotVariableRestorationrTrRra_deletersaveables_cache) rBrC save_pathr}r~restore_op_cacherrtr node_indexrYrZr.r.r/rDsN        z&_CheckpointRestoreCoordinator.__init__cCs|jSr2)rrBr.r.r/rc8sz,_CheckpointRestoreCoordinator.expect_partialcCs||_|j|dSr2)rrrgrfr.r.r/rc<scCs |j||jr||dSr2)rextendr)rBnew_opsr.r.r/new_restore_opsAs z-_CheckpointRestoreCoordinator.new_restore_opsNc Csg}|D]&}|jjdj}|j|j|q|s8|rt|}t || |j |j }tst|D]*\} } || | |jvsJ| |j| <qn|S)aRun or build restore operations for SaveableObjects. Args: tensor_saveables: `SaveableObject`s which correspond to Tensors. python_positions: List of CheckpointPositions bound to `PythonState` objects which must be restored eagerly. registered_savers: a dict mapping saver names-> object name -> Trackable. Returns: When graph building, a list of restore operations, either cached or newly created, to restore `tensor_saveables`. r) object_protorkcheckpoint_key trackable deserializer~ get_tensorr$validate_and_slice_inputsrMultiDeviceSaverrr}rtr executing_eagerlysortedrNrMr) rBZtensor_saveablesZpython_positionsregistered_saversrpositionkeyZflat_saveablesrname restore_opr.r.r/restore_saveablesFs(  z/_CheckpointRestoreCoordinator.restore_saveables)N) r[r\r]r^rDr`rcsetterrrr.r.r.r/rqsY  rqc@s*eZdZdZd ddZddZddZdS) _NameBasedRestoreCoordinatorz4Keeps the status of a name-based checkpoint restore.NcCs$||_||_t|_t|_dSr2)rrr'ObjectIdentityWeakKeyDictionaryrerrurv)rBrrr.r.r/rDns z%_NameBasedRestoreCoordinator.__init__c cs|D]\}}t|rTz |}WqXtyP|j|g|Yq YqX0n|}tj|gdd}|D]"\}}tj ||dD] }|Vqqpq dS)aCreate globally named SaveableObjects from attributes. If an object's attribute has no global name specified (default construction for the SaveableObject factory), records the failure in `self.unused_attributes` (which can then be used to make status assertions fail; see `NameBasedSaverStatus`). Args: trackable: An object to save. Yields: SaveableObjects for `trackable`'s attributes. F)convert_variable_to_tensor)oprN) _gather_saveables_for_checkpointrNcallable TypeErrorrerrMr$op_list_to_dictsaveable_objects_for_op) rBrrmsaveable_factorysaveablenames_to_saveablesrrsaveable_objectr.r.r/ globally_named_object_attributesws*     z=_NameBasedRestoreCoordinator.globally_named_object_attributesc Csts J||D]}g}d}|jD]}|j|jvrtd@tj |j |jgdg|j|jgd|jfd\}Wdn1s0Y| t |q(d}q(|r|j|g |jq|j|ddqdS) z.Runs restore ops for `trackable`'s attributes.Fzcpu:0z%s_checkpoint_read)r: tensor_namesshape_and_slicesrrNT)restored_tensorsrestored_shapes)r rrspecsrrrdeviceio_ops restore_v2rrMridentityrerr)rBrrrZtensor_missingspecZrestoredr.r.r/ eager_restores*      &z*_NameBasedRestoreCoordinator.eager_restore)N)r[r\r]r^rDrrr.r.r.r/rks $rc Ks t|}t|}t|dur@tj|||d\}}n t | }|j }|rj|durdt d|} nbt |t tjr||d}|durdn|} dt|jvrtj|| ||d} ntj|| |d} tjf| ||dd|WdS1s0YdS) zDA pared-down version of get_variable which does not reuse variables.N)rshapedtypez3If initializer is a constant, do not specify shape.rpartition_info)rrT) initial_valuerr use_resource)ras_dtyperas_shaper init_scoper_get_default_variable_store_get_default_initializerr base_dtype ValueError isinstancetyper Initializeras_listr) getargspecargs functoolspartialr VariableV1) rrr initializerrkwargsZ shape_objectinitializing_from_valuevariable_dtyperZ shape_listr.r.r/_default_gettersJ     rTcCs|j||||t|dS)z6Add a variable to a Trackable with no scope influence.)rrrrgetter trainable) _add_variable_with_custom_getterr)rrrrrrr.r.r/ add_variablesrc Cs^t|}z|tj}Wn,tjyFtd|dtjdYn0t }| ||S)aRetrieves information about the objects in a checkpoint. Example usage: ```python object_graph = tf.contrib.checkpoint.object_metadata( tf.train.latest_checkpoint(checkpoint_directory)) ckpt_variable_names = set() for node in object_graph.nodes: for attribute in node.attributes: ckpt_variable_names.add(attribute.full_name) ``` Args: save_path: The path to the checkpoint, as returned by `save` or `tf.train.latest_checkpoint`. Returns: A parsed `tf.contrib.checkpoint.TrackableObjectGraph` protocol buffer. Raises: ValueError: If an object graph was not found in the checkpoint. zThe specified checkpoint "zS" does not appear to be object-based (saved with TF2) since it is missing the key "zg". Likely it was created with the TF1 name-based saver and does not contain an object dependency graph.) r"rrr OBJECT_GRAPH_PROTO_KEYr NotFoundErrorrrTrackableObjectGraphParseFromString)rr~object_graph_stringrCr.r.r/object_metadatas   rcCstt|S)aTraverse the object graph and list all accessible objects. Looks for `Trackable` objects which are dependencies of `root_trackable`. Includes slot variables only if the variable they are slotting for and the optimizer are dependencies of `root_trackable` (i.e. if they would be saved with a checkpoint). Args: root_trackable: A `Trackable` object whose dependencies should be flattened. Returns: A flat list of objects. )r list_objectsgraph_view_libObjectGraphView)root_trackabler.r.r/r"srcCst|}dd|DS)aTraverse the object graph and find initialization ops. Looks for `Trackable` objects which are dependencies of `root_trackable` and which have an `initializer` property. Includes initializers for slot variables only if the variable they are slotting for and the optimizer are dependencies of `root_trackable` (i.e. if they would be saved with a checkpoint). Args: root_trackable: A `Trackable` object to gather initializers for. Returns: A list of initialization ops. cSs&g|]}t|dr|jdur|jqS)rN)hasattrr.0cr.r.r/ Csz'gather_initializers..)r)rtrackable_objectsr.r.r/gather_initializers3src#sLjjdfdd }t|dVWdn1s>0YdS)azCapture variables created within this scope as `Template` dependencies. Requires that `template.variable_scope` is active. This scope is intended as a compatibility measure, allowing a trackable object to add dependencies on variables created in a block of code which is not aware of object-based saving (and instead uses variable names heavily). This is how `Template` objects add dependencies on variables and sub-`Template`s. Where possible, use `tf.compat.v1.make_template` directly. Args: template: The `Template` object to register dependencies with. Yields: None (when used as a context manager). Nc sfdd}durrtdd}|sXjf|||dfd|S|\}}j||tddddf|fd|S) a{A variable creation hook which adds Trackable dependencies. Set for example during a `Template`'s first wrapped function execution. Ensures that (a) `template` depends on any trackable objects using their own `capture_dependencies` scope inside this scope which create variables, and (b) that any variables not in a more deeply nested scope are added as dependencies directly. The `trackable_parent` argument is passed between custom creators but ignored when the variable object itself is created. This argument indicates (if not `None`) that a more deeply nested scope has already added the variable as a dependency, and that parent scopes should add a dependency on that object rather than on the variable directly. Args: next_creator: See `variable_scope.variable_creator_scope`; the next creator in the chain. name: The (full, scope-influenced) name of the variable. The `name_prefix` itself is stripped for the purposes of object-based dependency tracking, but scopes opened within this scope are respected. initial_value: See `variable_scope.variable_creator_scope`. Taken explicitly so the argument can be re-named and used with `Trackable._add_variable_with_custom_getter`. trackable_parent: If not None, a more deeply nested trackable object and its name prefix which were passed to `capture_dependencies` to add a dependency on (rather than depending on the variable directly). **kwargs: Passed through to the next creator. Returns: The output of `next_creator`: the fetched/created variable object. cs|df|d|S)Nr)rr)pop)r inner_kwargsr next_creatorr.r/'_call_next_creator_renaming_initializers zhcapture_dependencies.._trackable_custom_creator.._call_next_creator_renaming_initializerNT)rrr overwritetrackable_parent)rr)rrr) startswithlenr_track_trackable) rrrrrrZscope_stripped_nameZ parent_objectZparent_name_prefixZ name_prefixtemplaterr/_trackable_custom_creator^s4%  z7capture_dependencies.._trackable_custom_creator)N)rrvariable_creator_scope)rrr.rr/capture_dependenciesJs C rc@sbeZdZdZejddZejddZejddZejdd d Z ejdd d Z d dZ dS) _LoadStatusz(Abstract base for load status callbacks.cCsdS)zCRaises an exception unless a non-trivial restoration has completed.Nr.rr.r.r/assert_consumedsz_LoadStatus.assert_consumedcCsdS)zERaises an exception unless existing Python objects have been matched.Nr.rr.r.r/assert_existing_objects_matchedsz+_LoadStatus.assert_existing_objects_matchedcCsdS)4Raises an exception if only the root object matched.Nr.rr.r.r/assert_nontrivial_matchsz#_LoadStatus.assert_nontrivial_matchNcCsdS)zBRuns restore ops from the checkpoint. Requires a valid checkpoint.Nr.rBr r.r.r/run_restore_opssz_LoadStatus.run_restore_opscCsdS)z?Runs restore ops from the checkpoint, or initializes variables.Nr.rr.r.r/initialize_or_restoresz!_LoadStatus.initialize_or_restorecCs|S)6Silence warnings about incomplete checkpoint restores.r.rr.r.r/rcsz_LoadStatus.expect_partial)N)N) r[r\r]r^abcabstractmethodrrrrrrcr.r.r.r/rs     rz'__internal__.tracking.streaming_restorecsNtr dSdurtttr,tdjdfddj_dS)aWhen graph building, runs restore ops as soon as they come in. Args: status: A _LoadStatus objects from an object-based saver's restore(). Streaming restore from name-based checkpoints is not currently supported. session: A session to run new restore ops in. NzStreaming restore not supported from name-based checkpoints when graph building. File a feature request if this limitation bothers you. As a workaround, consider either using tf.train.Checkpoint to load name-based checkpoints or enabling eager execution.r csj|jdS)N feed_dict)run _feed_dictrr statusr.r/z#streaming_restore..) r rr6rNameBasedSaverStatusNotImplementedErrorr _checkpointr)r r r.r r/streaming_restores    rcCsdd|DS)zHFilters out objects with no direct variable dependencies for assertions.cSsg|]}|r|qSr.)r)ror.r.r/rrz,_objects_with_attributes..r.)Z full_listr.r.r/_objects_with_attributessrc@sLeZdZdZddZddZddZdd Zdd d Zdd dZ ddZ d S)CheckpointLoadStatusa'Checks the status of checkpoint loading and manages restore ops. Returned from `Saver.restore`. Since `restore` may defer the loading of values in the checkpoint which don't yet have corresponding Python objects, `CheckpointLoadStatus` provides a callback to verify that checkpoint loading is complete (`assert_consumed`). When graph building, `restore` does not run restore ops itself since their creation may be deferred. The `run_restore_ops` method must be called once all Python objects with values to restore have been created and added to the dependency graph (this does not necessarily have to be the whole checkpoint; calling `run_restore_ops` while `assert_consumed` fails is supported and will partially restore the checkpoint). See `Saver.restore` for usage examples. cCs||_||_||_|j|_dSr2)rr _object_graph_viewroot_root)rB checkpointr rr.r.r/rDszCheckpointLoadStatus.__init__c Cst|jj}|t|jjjD]B\}}|js2q"|jj|d}|dur"t d|j |d|q"|jj rt d|jj |jj rg}t |jj D]4\}}|jj|}||j |d|d|qd|}t d||S) aAsserts that all objects in the checkpoint have been created/matched. Returns: `self` for chaining. Raises: AssertionError: If there are any Python objects in the dependency graph which have not been restored from this checkpoint or a later `restore`, or if there are any checkpointed values which have not been matched to Python objects. Nz Unresolved object in checkpoint : zUnresolved slot restorations: z (z):  zcUnused attributes in these objects (the attributes exist in the checkpoint but were not restored): )r?rrCrrPrIrkrygetAssertionErrorrXrresix iteritemsrMrO) rBrlrKrYrZunused_attribute_messages attributerVZjoined_attribute_messagesr.r.r/rsH     z$CheckpointLoadStatus.assert_consumedc Cst|jjjD]>\}}|jj|d}|dur|j|jjkrtd|dqt |j D](}t |t jrt|stqZ|jj|qZtt|jjt|jj}|rtt|}td|}td|d|d|dt|d||S) ayAsserts that trackable Python objects have been matched. Note that this is a weaker assertion than `assert_consumed`. It will only fail for existing Python objects which are (transitive) dependencies of the root object and which do not have an entry in the checkpoint. It will not fail, for example, if a `tf.keras.Layer` object has not yet been built and so has not created any `tf.Variable` objects. Returns: `self` for chaining. Raises: AssertionError: If a Python object exists in the transitive dependencies of the root object but does not have a value in the checkpoint. NzObject z& not assigned a value from checkpoint. zFound zq Python objects that were not bound to checkpointed values, likely due to changes in the Python program. Showing z of z unmatched objects: )rPrrCrIryr _update_uidrvrrrrrr!ZTrackableDataStructure_trackable_childrenr|addr'ObjectIdentitySetrvaluesrlistmin)rBrKrYrtrackable_objectunused_python_objectsZnum_unused_python_objectsZnum_variables_to_showr.r.r/r+sN     z4CheckpointLoadStatus.assert_existing_objects_matchedcCst|jD]}|jj|q t|jjdkrt t |jjt |jj }|rlt dt |nt d|jjd|S)rrzNothing except the root object matched a checkpointed value. Typically this means that the checkpoint does not match the Python program. The following objects have no matching checkpointed value: z4Nothing to load. No dependencies have been added to z yet.)rrrrr|r%rryr'r&rr'rr(r)rBr*r+r.r.r/r\s.  z,CheckpointLoadStatus.assert_nontrivial_matchNcCs2tr dS|durt}|j|jj|jddS)z:Run operations to restore objects in the dependency graph.Nr)r rr6r rrr rr.r.r/rrs z$CheckpointLoadStatus.run_restore_opscsftr dS|durt}tj}tjj fdd|D}j |d| |dS)aURun operations to initialize or restore objects in the dependency graph. Any objects in the dependency graph which have initializers but are not in the checkpoint will have those initializers run, unless those variables are being restored by a later call to `tf.train.Checkpoint.restore()`. This method has a sibling in `InitializationOnlyStatus` which instead initializes variables. That type is returned if no checkpoint is specified in `Saver.restore`. Args: session: The session to run init/restore ops in. If `None`, uses the default session. Ncs@g|]8}t|dr|vrt|djjdjjkr|jqS)rr#r)rgetattrrrvrrZalready_initialized_objectsrBr.r/rs z>CheckpointLoadStatus.initialize_or_restore..r ) r rr6rrrr'r&rryr'rr )rBr Z all_objectsZ'initializers_for_non_restored_variablesr.r-r/rzs    z*CheckpointLoadStatus.initialize_or_restorecCs d|j_|S)rT)rrcrr.r.r/rcsz#CheckpointLoadStatus.expect_partial)N)N) r[r\r]r^rDrrrrrrcr.r.r.r/rs+1  rc@sDeZdZdZddZddZddZdd Zdd d Zdd dZ d S)InitializationOnlyStatusa^Returned from `Saver.restore` when no checkpoint has been specified. Objects of this type have the same `assert_consumed` method as `CheckpointLoadStatus`, but it always fails. However, `initialize_or_restore` works on objects of both types, and will initialize variables in `InitializationOnlyStatus` objects or restore them otherwise. cCs||_||_|j|_dSr2) _restore_uidrrr)rBobject_graph_viewrvr.r.r/rDsz!InitializationOnlyStatus.__init__cCs tddSzDAssertion for consistency with `CheckpointLoadStatus`. Always fails.zDNo checkpoint specified (save_path=None); nothing is being restored.Nrrr.r.r/rsz(InitializationOnlyStatus.assert_consumedcCs tddSr1r2rr.r.r/rsz8InitializationOnlyStatus.assert_existing_objects_matchedcCs tddSr1r2rr.r.r/rsz0InitializationOnlyStatus.assert_nontrivial_matchNcCs tddS)zFor consistency with `CheckpointLoadStatus`. Use `initialize_or_restore` for initializing if no checkpoint was passed to `Saver.restore` and restoring otherwise. Args: session: Not used. z[No checkpoint specified, so no restore ops are available (save_path=None to Saver.restore).Nr2rr.r.r/rs z(InitializationOnlyStatus.run_restore_opscsFtr dS|durt}tj}fdd|D}||dS)aRuns initialization ops for variables. Objects which would be saved by `Saver.save` will be initialized, unless those variables are being restored by a later call to `tf.train.Checkpoint.restore()`. This method does nothing when executing eagerly (initializers get run eagerly). Args: session: The session to run initialization ops in. If `None`, uses the default session. Ncs>g|]6}t|dr|jdurt|djdjkr|jqS)rNr#r)rrr,r/rrr.r/rs zBInitializationOnlyStatus.initialize_or_restore..)r rr6rrrr )rBr r initializersr.rr/rs  z.InitializationOnlyStatus.initialize_or_restore)N)N) r[r\r]r^rDrrrrrr.r.r.r/r.s  r.aRestoring a name-based tf.train.Saver checkpoint using the object-based restore API. This mode uses global names to match variables, and so is somewhat fragile. It also adds new restore ops to the graph each time it is called when graph building. Prefer re-encoding training checkpoints in the object-based format: run save() on the object-based saver (the same one this message is coming from) and use that checkpoint in the future.c@sbeZdZdZejdedddZddZdd Z d d Z d d Z ddZ dddZ dddZdS)rz4Status for loading a name-based training checkpoint.N)date instructionscCs||_||_g|_|j|_dSr2)rr_optionally_restoredrr)rBrr0r.r.r/rDszNameBasedSaverStatus.__init__cCs|j|dS)aAdd a variable to the list of optionally restored variables. There are situations where certain variables should be ignored in assertions such as assert_existing_objects_matched(). One example is that of a checkpoint saved with train.Saver(), and restored with train.Checkpoint(): it is possible for the train.Saver() checkpoint to be missing the internal `save_counter` variable, which we want to ignore on restore. Args: var: The variable to treat as optionally restored. N)r6rM)rBvarr.r.r/add_to_optionally_restoreds z/NameBasedSaverStatus.add_to_optionally_restoredcs|tjj}fdd|D}|rBdd|D}td|tjD](}||j jj krNtd|qNS)z3Raises an exception if any variables are unmatched.cs(g|] tfddjDrqS)c3s|]}d|uVqdS)rNr.)rxar.r/ rzBNameBasedSaverStatus.assert_consumed...)allr6)rrr:r/rsz8NameBasedSaverStatus.assert_consumed..cSs g|]\}}d|d|qS)z rr.)rrVrkr.r.r/rsz5Some objects had attributes which were not restored: zObject not restored: ) r(rrerNrrrr_maybe_initialize_trackabler#rv)rBreZunused_attribute_stringsrr.rr/r s$ z$NameBasedSaverStatus.assert_consumedcCs|Sz?Raises an exception if currently created objects are unmatched.rrr.r.r/r"sz4NameBasedSaverStatus.assert_existing_objects_matchedcCs|Sr?r@rr.r.r/r*sz,NameBasedSaverStatus.assert_nontrivial_matchcCsTt|j}g}|D]:}||j|jjkr|jj|_nq||j|q|S)z>Walk the object graph, using global names for SaveableObjects.) rrrr>r#rrvrr)rBobjectssaveable_objectsrr.r.r/_gather_saveable_objects2s   z-NameBasedSaverStatus._gather_saveable_objectscCshtr dS|durt}td0|}t|j||j j dWdn1sZ0YdS)zFLoad the name-based checkpoint using a new `tf.compat.v1.train.Saver`.N/cpu:0)sessr) r rr6rrrC v1_saver_libSaverrrr)rBr saveablesr.r.r/rBs  z$NameBasedSaverStatus.run_restore_opscCs|j|ddS)zAlias for `run_restore_ops`.r N)rrr.r.r/rMsz*NameBasedSaverStatus.initialize_or_restore)N)N)r[r\r]r^r& deprecated _DEPRECATED_RESTORE_INSTRUCTIONSrDr8rrrrCrrr.r.r.r/rs  rc@s"eZdZdZddZdddZdS)_SessionWithFeedDictAdditionsz7Pretends to be a session, inserts extra feeds on run().cCs||_||_dSr2)_wrapped_session_feed_additions)rBr feed_additionsr.r.r/rDUsz&_SessionWithFeedDictAdditions.__init__NcKs:|duri}n|}||j|jjf||d|S)N)fetchesr )copyupdaterMrLr )rBrOr rr.r.r/r Ys z!_SessionWithFeedDictAdditions.run)N)r[r\r]r^rDr r.r.r.r/rKRsrKc@s@eZdZdZddZdddZddd Zdd d Zdd d ZdS)TrackableSaveraSaves and restores a `Trackable` object and its dependencies. See `Trackable` for details of dependency management. `Saver` wraps `tf.compat.v1.train.Saver` for saving, including extra information about the graph of dependencies between Python objects. When restoring, it uses this information about the save-time dependency graph to more robustly match objects with their checkpointed values. When executing eagerly, it supports restoring variables on object creation (see `Saver.restore`). Values in a checkpoint are mapped to `Trackable` Python objects (`Variable`s, `Optimizer`s, `Layer`s) based on the names provided when the checkpoint was written. To avoid breaking existing checkpoints when modifying a class, dependency names (the names of attributes to which `Trackable` objects are assigned) may not change. These names are local to objects, in contrast to the `Variable.name`-based save/restore from `tf.compat.v1.train.Saver`, and so allow additional program transformations. cCsN||_trd|_n t|_d|_d|_d|_d|_ d|_ i|_ d|_ dS)zConfigure saving. Args: graph_view: An `ObjectGraphView` object containing a description of the object graph to save. N) _graph_viewr r_saveables_cacher'r_file_prefix_placeholder_object_graph_feed_tensor_last_save_object_graph_file_prefix_feed_tensor_cached_save_operation_restore_op_cache _object_map)rBrr.r.r/rDxs zTrackableSaver.__init__NcCstj|j|j|jd\}}}}|durftd$tj| t j d}Wdqx1sZ0Yn| || it j|vsJ|t j|t jd||||fS)z@Wraps _serialize_object_graph to include the object graph proto.)r object_maprNrDr)tensorr)rZserialize_gathered_objectsrSr[rTrrrconstantSerializeToStringrstringrQr rrMNoRestoreSaveable)rBobject_graph_tensornamed_saveable_objects graph_protorNrr.r.r/_gather_saveabless.   & z TrackableSaver._gather_saveablesFcsj|d\fddfdd}fdd}jr|tdurdttj|d at_jfSS) aCreate or retrieve save ops. Args: file_prefix: The prefix for saved checkpoint files. object_graph_tensor: A `Tensor` to which the current object graph will be fed. options: `CheckpointOptions` object. update_ckpt_state: Optional bool flag. Indiciate whether the internal checkpoint state needs to be updated. Returns: A two-element tuple with a filename tensor and a feed_dict of tensors to feed when running it (if graph building). The feed dict contains the current object graph and any Python state to be saved in the checkpoint. When executing eagerly only the first argument is meaningful. )rbc sjkststrt}|jd}tdHt |gt _ Wdn1sn0YWdn1s0Y_j fS)z1Create and execute the SaveOp for the checkpoint.rtrDN) rWr rrinside_functionrrsavercontrol_dependenciesrrrY)r#save_op)rN file_prefixrdrcrtrrBr.r/ _run_saves  HzBTrackableSaver._save_cached_when_graph_building.._run_savec sD]}|j}t|x|jD]`}|j}|j}|dur tt|(t||_t||_Wdq 1sv0Yq Wdq1s0YqdS)z(Copy the tensors to the host CPU device.N) rrrr]r$set_cpu0rr_tensor)roriginal_devicerr]r)rcr.r/ _copy_tensorss   zFTrackableSaver._save_cached_when_graph_building.._copy_tensorscsNttjddd&r,ttWdn1s@0YdS)z8The thread function for executing async checkpoint save.F) enable_asyncenable_streaming_enqueueN)r executor_scoper new_executor!_update_checkpoint_state_internal#_convert_file_name_tensor_to_stringr.)rlrkupdate_ckpt_stater.r/_async_save_fnszGTrackableSaver._save_cached_when_graph_building.._async_save_fnN)target)re$experimental_enable_async_checkpoint_ASYNC_CHECKPOINT_THREADrO threadingThreadstartrY)rBrkrbrtrwrprxr.) rlrNrkrdrcrtrrBrwr/ _save_cached_when_graph_buildings"    z/TrackableSaver._save_cached_when_graph_buildingc Cs^|p t}i}t o"t }|r4d||f}|r|jdurtd4tj dt j d|_tj dt j d|_ Wdn1s0Y|j}|j } ||| <n>td tj |t j d} Wdn1s0Yd}t|sttj||| |||\} } | r&|| |s2d}n|durBt}|rV|j| |dS| SdS)aSave a training checkpoint. The saved checkpoint includes variables created by this object and any Trackable objects it depends on at the time `Saver.save()` is called. Args: file_prefix: A prefix to use for the checkpoint filenames (/path/to/directory/and_a_prefix). Names are generated based on this prefix and `checkpoint_number`, if provided. checkpoint_number: An integer variable or Tensor, used to number checkpoints. Typically this value is saved along with other variables in training checkpoints, which will happen automatically if it was created by `root_trackable` or one of its dependencies (via `Trackable._add_variable`). session: The session to evaluate variables in. Ignored when executing eagerly. If not provided when graph building, the default session is used. options: Optional `tf.train.CheckpointOptions` object. update_ckpt_state: Optional bool flag. Indiciate whether the internal checkpoint state needs to be updated. Set this to True only if calling from tf.train.Checkpoint.save() to enable updating the checkpoint state. By default this is set to False, i.e., not updating checkpoint state. Returns: The full path to the checkpoint. %s-%dNrDrrr)rCheckpointOptionsr rrrgrVrrr^rr`rXconvert_to_tensorr is_tensorrrecursive_create_dirospathdirnamerrQr6r ) rBrkcheckpoint_numberr rtrwr Z use_sessionrbfile_prefix_tensorrZnew_feed_additionsr.r.r/rhsL     &  $   zTrackableSaver.savec CsP|p t}|dur$t|jtStdur4tt |}t }|rRd}n| }z| tj}Wnbtjyt||d}|st|jD]"}||j|||qt||jdYS0|r*|jdurtdtd|_Wdn1s0Y|j} |j|i} n:tdt|} Wdn1sV0Yd} t} | |t!| || ||j"|j||j#d} t$j%| dd&|jj'|jj(r<|jj(D]|} | j)d krҐqd}| j*dj+D] }|j,| j)kr|j-}qq|| j.vrq|dur"qt$j%| |d&| j/qt0| |j| d }|S) ak Restore a training checkpoint. Restores `root_trackable` and any objects that it tracks (transitive). Either assigns values immediately if variables to restore have been created already, or defers restoration until the variables are created. Dependencies added to the `root_trackable` passed to the constructor after this call will be matched if they have a corresponding object in the checkpoint. When building a graph, restorations are added to the graph but not run. ```python saver = Saver(root) saver.restore(path) ``` To ensure that loading is complete and no more deferred restorations will take place, you can use the `assert_consumed()` method of the status object returned by the `restore` call. The assert will raise an exception unless every object was matched and all checkpointed values have a matching variable object. ```python saver = Saver(root) saver.restore(path).assert_consumed() ``` When graph building, `assert_consumed()` indicates that all of the restore ops which will be created for this checkpoint have been created. They can be run via the `run_restore_ops()` function of the status object: ```python saver.restore(path).assert_consumed().run_restore_ops() ``` If the checkpoint has not been consumed completely, then the list of restore ops will grow as more objects are added to the dependency graph. Name-based `tf.compat.v1.train.Saver` checkpoints can be loaded using this method. There is no deferred loading, and names are used to match variables. No restore ops are created/run until `run_restore_ops()` or `initialize_or_restore()` are called on the returned status object, even when executing eagerly. Re-encode name-based checkpoints using this object-based `Saver.save` as soon as possible. Args: save_path: The path to the checkpoint, as returned by `save` or `tf.train.latest_checkpoint`. If None (as when there is no latest checkpoint for `tf.train.latest_checkpoint` to return), returns an object which may run initializers for objects in the dependency graph. If the checkpoint was written by the name-based `tf.compat.v1.train.Saver`, names are used to match variables. options: Optional `tf.train.CheckpointOptions` object. Returns: A load status object, which can be used to make assertions about the status of checkpoint restoration and run initialization/restore ops (of type `CheckpointLoadStatus`, or `InitializationOnlyStatus` if `save_path` is `None`). If `save_path` points to a name-based checkpoint, a `NameBasedSaverStatus` object is returned which runs restore ops from a name-based saver. Raises: RuntimeError: When a checkpoint file saved by async checkpoint is not available upon restore(). N)rr)r0rDmodel)rCrr}r~rrrtrr)rproto_idr)rr )1rrr.rSrrur{rOr"rr rrrr rrrrrrr>_name_based_restoresr%_name_based_attribute_restorerrUrrr^rrrrqrZrT restore_libZCheckpointPositionrrattached_dependenciesrrIrJrLrKryrefr)rBrrtr~graph_buildingrrZrestore_coordinatorZexisting_trackablerZfile_prefix_feed_dictrCrrrZ proto_refZ load_statusr.r.r/rVsE        ,  *        zTrackableSaver.restore)N)F)NNNF)N) r[r\r]r^rDrerrhrr.r.r.r/rRcs&  ^ DrRcCs tt|\}}t||S)apCreates a static `tf.compat.v1.train.Saver` from a trackable object. The returned `Saver` saves object-based checkpoints, but these checkpoints will no longer reflect structural changes to the object graph, only changes to the values of `Variable`s added as dependencies of the root object before `freeze` was called. `restore` works on the returned `Saver`, but requires that the object graph of the checkpoint being loaded exactly matches the object graph when `freeze` was called. This is in contrast the object-based restore performed by `tf.train.Checkpoint` which attempts a fuzzy matching between a checkpoint's object graph and the current Python object graph. Args: root_trackable: A trackable object to save. Returns: A saver which saves object-based checkpoints for the object graph frozen at the time `frozen_saver` was called. )rZfrozen_saveables_and_saversrrrr)rrcrr.r.r/ frozen_saversrcCs,t|tjtjfs(td|d|ddS)Nz`Checkpoint` was expecting zD to be a trackable object (an object derived from `Trackable`), got . If you believe this object should be trackable (i.e. it is part of the TensorFlow Python API and manages state), please open an issue.)rr Trackabler Functionr)rVrr.r.r/_assert_trackables rcCs tjtj|||gdddS)z!Update internal checkpoint state.Tsave_dirmodel_checkpoint_pathall_model_checkpoint_pathssave_relative_pathsN)r update_checkpoint_state_internalrrr) file_pathr.r.r/ru!s  rucCs4|}t|r&tr0t|}n t|}|S)z#Convert file name tensor to string.)r is_tf_typer rr%as_strnumpy)r]outputr.r.r/rv*s   rvztrain.CheckpointcsPeZdZdZfddZddZdddZed d Zdd d Z d dZ Z S) CheckpointV1aGroups trackable objects, saving and restoring them. `Checkpoint`'s constructor accepts keyword arguments whose values are types that contain trackable state, such as `tf.compat.v1.train.Optimizer` implementations, `tf.Variable`, `tf.keras.Layer` implementations, or `tf.keras.Model` implementations. It saves these values with a checkpoint, and maintains a `save_counter` for numbering checkpoints. Example usage when graph building: ```python import tensorflow as tf import os checkpoint_directory = "/tmp/training_checkpoints" checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model) status = checkpoint.restore(tf.train.latest_checkpoint(checkpoint_directory)) train_op = optimizer.minimize( ... ) status.assert_consumed() # Optional sanity checks. with tf.compat.v1.Session() as session: # Use the Session to restore variables, or initialize them if # tf.train.latest_checkpoint returned None. status.initialize_or_restore(session) for _ in range(num_training_steps): session.run(train_op) checkpoint.save(file_prefix=checkpoint_prefix) ``` Example usage with eager execution enabled: ```python import tensorflow as tf import os tf.compat.v1.enable_eager_execution() checkpoint_directory = "/tmp/training_checkpoints" checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model) status = checkpoint.restore(tf.train.latest_checkpoint(checkpoint_directory)) for _ in range(num_training_steps): optimizer.minimize( ... ) # Variables will be restored on creation. status.assert_consumed() # Optional sanity checks. checkpoint.save(file_prefix=checkpoint_prefix) ``` `Checkpoint.save` and `Checkpoint.restore` write and read object-based checkpoints, in contrast to `tf.compat.v1.train.Saver` which writes and reads `variable.name` based checkpoints. Object-based checkpointing saves a graph of dependencies between Python objects (`Layer`s, `Optimizer`s, `Variable`s, etc.) with named edges, and this graph is used to match variables when restoring a checkpoint. It can be more robust to changes in the Python program, and helps to support restore-on-create for variables when executing eagerly. Prefer `tf.train.Checkpoint` over `tf.compat.v1.train.Saver` for new code. `Checkpoint` objects have dependencies on the objects passed as keyword arguments to their constructors, and each dependency is given a name that is identical to the name of the keyword argument for which it was created. TensorFlow classes like `Layer`s and `Optimizer`s will automatically add dependencies on their variables (e.g. "kernel" and "bias" for `tf.keras.layers.Dense`). Inheriting from `tf.keras.Model` makes managing dependencies easy in user-defined classes, since `Model` hooks into attribute assignment. For example: ```python class Regress(tf.keras.Model): def __init__(self): super(Regress, self).__init__() self.input_transform = tf.keras.layers.Dense(10) # ... def call(self, inputs): x = self.input_transform(inputs) # ... ``` This `Model` has a dependency named "input_transform" on its `Dense` layer, which in turn depends on its variables. As a result, saving an instance of `Regress` using `tf.train.Checkpoint` will also save all the variables created by the `Dense` layer. When variables are assigned to multiple workers, each worker writes its own section of the checkpoint. These sections are then merged/re-indexed to behave as a single checkpoint. This avoids copying all variables to one worker, but does require that all workers see a common filesystem. While `tf.keras.Model.save_weights` and `tf.train.Checkpoint.save` save in the same format, note that the root of the resulting checkpoint is the object the save method is attached to. This means saving a `tf.keras.Model` using `save_weights` and loading into a `tf.train.Checkpoint` with a `Model` attached (or vice versa) will not match the `Model`'s variables. See the [guide to training checkpoints](https://www.tensorflow.org/guide/checkpoint) for details. Prefer `tf.train.Checkpoint` over `tf.keras.Model.save_weights` for training checkpoints. Attributes: save_counter: Incremented when `save()` is called. Used to number checkpoints. c stt|t tdur$taWdn1s80Yt|dddD]<\}}t|||t t ||t j t jfsVtd|dqVd|_d|_tt||_dS)aGroup objects into a training checkpoint. Args: **kwargs: Keyword arguments are set as attributes of this object, and are saved with the checkpoint. Values must be trackable objects. Raises: ValueError: If objects in `kwargs` are not trackable. NcSs|dSNrr.itemr.r.r/rrz'CheckpointV1.__init__..rzX`Checkpoint` was expecting a trackable object (an object derived from `Trackable`), got r)superrrD_END_TIME_OF_LAST_WRITE_LOCK_END_TIME_OF_LAST_WRITEtimerrNsetattrrr,r rr rr _save_counter_save_assign_oprRrr_saver)rBrkv __class__r.r/rDs" & zCheckpointV1.__init__c CsT|jdurPtd,tt|ddtjdd|_Wdn1sF0YdS)/Create a save counter if it does not yet exist.NrD save_counterrFrrrr)rrrr! NoDependencyrrint64rr.r.r/_maybe_create_save_counters  z'CheckpointV1._maybe_create_save_counterNcCst}|jj||d}t}tjtt||dt(tjttt |d|a Wdn1sf0Yt |rt rt|}n t|}tjtt|d|S)aAWrites a training checkpoint. The checkpoint includes variables created by this object and any trackable objects it depends on at the time `Checkpoint.write()` is called. `write` does not number checkpoints, increment `save_counter`, or update the metadata used by `tf.train.latest_checkpoint`. It is primarily intended for use by higher level checkpoint management utilities. `save` provides a very basic implementation of these features. Args: file_prefix: A prefix to use for the checkpoint filenames (/path/to/directory/and_a_prefix). session: The session to evaluate variables in. Ignored when executing eagerly. If not provided when graph building, the default session is used. Returns: The full path to the checkpoint (i.e. `file_prefix`). )rkr  api_label microsecondsNrfilesize)rrrhrAddCheckpointWriteDuration_CHECKPOINT_V1r0rAddTrainingTimeSavedrrrr rr%rrRecordCheckpointSizer>)rBrkr start_timerend_timer.r.r/writes."  zCheckpointV1.writecCs||jSzAn integer variable which starts at zero and is incremented on save. Used to number checkpoints. Returns: The save counter variable. rrrr.r.r/rs zCheckpointV1.save_countercCst }|rDtrtd|dur,t}|jdurD||jj |rR|j durt |j |jj ddd}Wdn1s0Y|rt ||_ |r||j }n|}|jd||f|d}tjtj|||gdd|S) aSaves a training checkpoint and provides basic checkpoint management. The saved checkpoint includes variables created by this object and any trackable objects it depends on at the time `Checkpoint.save()` is called. `save` is a basic convenience wrapper around the `write` method, sequentially numbering checkpoints using `save_counter` and updating the metadata used by `tf.train.latest_checkpoint`. More advanced checkpoint management, for example garbage collection and custom numbering, may be provided by other utilities which also wrap `write` (`tf.train.CheckpointManager` for example). Args: file_prefix: A prefix to use for the checkpoint filenames (/path/to/directory/and_a_prefix). Names are generated based on this prefix and `Checkpoint.save_counter`. session: The session to evaluate variables in. Ignored when executing eagerly. If not provided when graph building, the default session is used. Returns: The full path to the checkpoint. ?Calling tf.train.Checkpoint.save() from a function is not supported, as save() modifies saving metadata in ways not supported by TensorFlow Operations. Consider using tf.train.Checkpoint.write(), a lower-level API which does not update metadata. tf.train.latest_checkpoint and related APIs will not see this checkpoint.NrT read_valuerr r)r rrrgrr6rr rrr colocate_with assign_addr!rrrrrrrr)rBrkr r assign_oprrr.r.r/rhs8  .   zCheckpointV1.savecCsPt}|jj|d}|t|tr4||jtj t t |td|S)aRestore a training checkpoint. Restores this `Checkpoint` and any objects it depends on. When executing eagerly, either assigns values immediately if variables to restore have been created already, or defers restoration until the variables are created. Dependencies added after this call will be matched if they have a corresponding object in the checkpoint (the restore request will queue in any trackable object waiting for the expected dependency to be added). When graph building, restoration ops are added to the graph but not run immediately. ```python checkpoint = tf.train.Checkpoint( ... ) checkpoint.restore(path) ``` To ensure that loading is complete and no more deferred restorations will take place, you can use the `assert_consumed()` method of the status object returned by `restore`. The assert will raise an exception if any Python objects in the dependency graph were not found in the checkpoint, or if any checkpointed values do not have a matching Python object: ```python checkpoint = tf.train.Checkpoint( ... ) checkpoint.restore(path).assert_consumed() ``` When graph building, `assert_consumed()` indicates that all of the restore ops that will be created for this checkpoint have been created. They can be run via the `run_restore_ops()` method of the status object: ```python checkpoint.restore(path).assert_consumed().run_restore_ops() ``` If the checkpoint has not been consumed completely, then the list of restore ops will grow as more objects are added to the dependency graph. To check that all variables in the Python object have restored values from checkpoint, use `assert_existing_objects_matched()`. This assertion is useful when called after the variables in your graph have been created. Name-based `tf.compat.v1.train.Saver` checkpoints can be loaded using this method. Names are used to match variables. No restore ops are created/run until `run_restore_ops()` or `initialize_or_restore()` are called on the returned status object when graph building, but there is restore-on-creation when executing eagerly. Re-encode name-based checkpoints using `tf.train.Checkpoint.save` as soon as possible. Args: save_path: The path to the checkpoint, as returned by `save` or `tf.train.latest_checkpoint`. If None (as when there is no latest checkpoint for `tf.train.latest_checkpoint` to return), returns an object which may run initializers for objects in the dependency graph. If the checkpoint was written by the name-based `tf.compat.v1.train.Saver`, names are used to match variables. Returns: A load status object, which can be used to make assertions about the status of a checkpoint restoration and run initialization/restore ops. The returned status object has the following methods: * `assert_consumed()`: Raises an exception if any variables are unmatched: either checkpointed values which don't have a matching Python object or Python objects in the dependency graph with no values in the checkpoint. This method returns the status object, and so may be chained with `initialize_or_restore` or `run_restore_ops`. * `assert_existing_objects_matched()`: Raises an exception if any existing Python objects in the dependency graph are unmatched. Unlike `assert_consumed`, this assertion will pass if values in the checkpoint have no corresponding Python objects. For example a `tf.keras.Layer` object which has not yet been built, and so has not created any variables, will pass this assertion but will fail `assert_consumed`. Useful when loading part of a larger checkpoint into a new Python program, e.g. a training checkpoint with a `tf.compat.v1.train.Optimizer` was saved but only the state required for inference is being loaded. This method returns the status object, and so may be chained with `initialize_or_restore` or `run_restore_ops`. * `assert_nontrivial_match()`: Asserts that something aside from the root object was matched. This is a very weak assertion, but is useful for sanity checking in library code where objects may exist in the checkpoint which haven't been created in Python and some Python objects may not have a checkpointed value. * `expect_partial()`: Silence warnings about incomplete checkpoint restores. Warnings are otherwise printed for unused parts of the checkpoint file or object when the `Checkpoint` object is deleted (often at program shutdown). * `initialize_or_restore(session=None)`: When graph building, runs variable initializers if `save_path` is `None`, but otherwise runs restore operations. If no `session` is explicitly specified, the default session is used. No effect when executing eagerly (variables are initialized or restored eagerly). * `run_restore_ops(session=None)`: When graph building, runs restore operations. If no `session` is explicitly specified, the default session is used. No effect when executing eagerly (restore operations are run eagerly). May only be called when `save_path` is not `None`. )rr) rrrrrrr8rrAddCheckpointReadDurationrr0)rBrrr r.r.r/rJsn   zCheckpointV1.restore)N)N) r[r\r]r^rDrrr`rrhr __classcell__r.r.rr/r8sj  2 .rz1Cannot create a Checkpoint with keyword argument z if root.z already exists.)r)rrrDrrrrrrrwrrr>r!r_lookup_dependencyrrNrr,rMr WeakTrackableReferencerrRrrr_attached_dependencies) rBrrrZtrackable_rootrrZ converted_vrWrr.r/rD-sV&         zCheckpoint.__init__c Cs|jdurtdtt|ddtjdd|_|jdur|j t d|jt |j tjrj| }n|j }|jdd}|r|d|jWdn1s0YdS)rNrDrrFrr.)rrrr!rrrrrrMr TrackableReferencerrrwr_deferred_dependenciesrr)rBrrr.r.r/rrs*      z%Checkpoint._maybe_create_save_countercCs"t|tjrt|}|||S)aNWrites a training checkpoint. The checkpoint includes variables created by this object and any trackable objects it depends on at the time `Checkpoint.write()` is called. `write` does not number checkpoints, increment `save_counter`, or update the metadata used by `tf.train.latest_checkpoint`. It is primarily intended for use by higher level checkpoint management utilities. `save` provides a very basic implementation of these features. Checkpoints written with `write` must be read with `read`. Example usage: ``` step = tf.Variable(0, name="step") checkpoint = tf.Checkpoint(step=step) checkpoint.write("/tmp/ckpt") # Later, read the checkpoint with read() checkpoint.read("/tmp/ckpt") # You can also pass options to write() and read(). For example this # runs the IO ops on the localhost: options = tf.CheckpointOptions(experimental_io_device="/job:localhost") checkpoint.write("/tmp/ckpt", options=options) # Later, read the checkpoint with read() checkpoint.read("/tmp/ckpt", options=options) ``` Args: file_prefix: A prefix to use for the checkpoint filenames (/path/to/directory/and_a_prefix). options: Optional `tf.train.CheckpointOptions` object. Returns: The full path to the checkpoint (i.e. `file_prefix`). )rrPathLikefspath_write)rBrkrtr.r.r/rs)  zCheckpoint.writeFcCst}|pt}|jj|||d}t}tjtt||dt (tj ttt |d|a Wdn1st0Yt |}|j stjtt|d|S)aInternal method that implements Checkpoint.write(). Args: file_prefix: A prefix to use for the checkpoint filenames (/path/to/directory/and_a_prefix). options: Optional `tf.train.CheckpointOptions` object. update_ckpt_state: Optional bool flag. Indiciate whether the internal checkpoint state needs to be updated. Returns: The full path to the checkpoint (i.e. `file_prefix`). )rkrtrwrNr)rrrrrhrr_CHECKPOINT_V2r0rrrrvrzrr>)rBrkrtrwrrrr.r.r/rs4 "zCheckpoint._writecCs||jSrrrr.r.r/rs zCheckpoint.save_countercCs"t|tjrt|}|p t}t }|rl|jr>t dt rNt dt }|j durl||jj|rz|jdurt |j |jjddd}Wdn1s0Y|rt||_|r||j}n|}|jd||f|dd}|jst||s|jst|S) aSaves a training checkpoint and provides basic checkpoint management. The saved checkpoint includes variables created by this object and any trackable objects it depends on at the time `Checkpoint.save()` is called. `save` is a basic convenience wrapper around the `write` method, sequentially numbering checkpoints using `save_counter` and updating the metadata used by `tf.train.latest_checkpoint`. More advanced checkpoint management, for example garbage collection and custom numbering, may be provided by other utilities which also wrap `write` and `read`. (`tf.train.CheckpointManager` for example). ``` step = tf.Variable(0, name="step") checkpoint = tf.train.Checkpoint(step=step) checkpoint.save("/tmp/ckpt") # Later, read the checkpoint with restore() checkpoint.restore("/tmp/ckpt-1") # You can also pass options to save() and restore(). For example this # runs the IO ops on the localhost: options = tf.train.CheckpointOptions(experimental_io_device="/job:localhost") checkpoint.save("/tmp/ckpt", options=options) # Later, read the checkpoint with restore() checkpoint.restore("/tmp/ckpt-1", options=options) ``` Args: file_prefix: A prefix to use for the checkpoint filenames (/path/to/directory/and_a_prefix). Names are generated based on this prefix and `Checkpoint.save_counter`. options: Optional `tf.train.CheckpointOptions` object. Returns: The full path to the checkpoint. z6Async checkpoint is not supported for non-eager mode. rNrTrr)rtrw)rrrrrrr rrzrrrgr6rr rrrrrr!rrrru async_wait)rBrkrtrr rrrr.r.r/rhsD)     .   zCheckpoint.savecCsVt}t|tjrt|}|p(t}|jj||d}t j t t |td|S)aRReads a training checkpoint written with `write`. Reads this `Checkpoint` and any objects it depends on. This method is just like `restore()` but does not expect the `save_counter` variable in the checkpoint. It only restores the objects that the checkpoint already depends on. The method is primarily intended for use by higher level checkpoint management utilities that use `write()` instead of `save()` and have their own mechanisms to number and track checkpoints. Example usage: ```python # Create a checkpoint with write() ckpt = tf.train.Checkpoint(v=tf.Variable(1.)) path = ckpt.write('/tmp/my_checkpoint') # Later, load the checkpoint with read() # With restore() assert_consumed() would have failed. checkpoint.read(path).assert_consumed() # You can also pass options to read(). For example this # runs the IO ops on the localhost: options = tf.train.CheckpointOptions( experimental_io_device="/job:localhost") checkpoint.read(path, options=options) ``` Args: save_path: The path to the checkpoint as returned by `write`. options: Optional `tf.train.CheckpointOptions` object. Returns: A load status object, which can be used to make assertions about the status of a checkpoint restoration. See `restore` for details. )rrtr) rrrrrrrrrrrrr0)rBrrtrresultr.r.r/read^ s'    zCheckpoint.readc Cs|}t|tjrt|}|durVt|rVtt|sLtt |rVt |}z"|j ||d}t rvt WnDtjy}z*tddd|d|jdWYd}~n d}~00|t|tr||j|S)a}Restores a training checkpoint. Restores this `Checkpoint` and any objects it depends on. This method is intended to be used to load checkpoints created by `save()`. For checkpoints created by `write()` use the `read()` method which does not expect the `save_counter` variable added by `save()`. `restore()` either assigns values immediately if variables to restore have been created already, or defers restoration until the variables are created. Dependencies added after this call will be matched if they have a corresponding object in the checkpoint (the restore request will queue in any trackable object waiting for the expected dependency to be added). ```python checkpoint = tf.train.Checkpoint( ... ) checkpoint.restore(path) # You can additionally pass options to restore(): options = tf.CheckpointOptions(experimental_io_device="/job:localhost") checkpoint.restore(path, options=options) ``` To ensure that loading is complete and no more deferred restorations will take place, use the `assert_consumed()` method of the status object returned by `restore()`: ```python checkpoint.restore(path, options=options).assert_consumed() ``` The assert will raise an error if any Python objects in the dependency graph were not found in the checkpoint, or if any checkpointed values do not have a matching Python object. Name-based `tf.compat.v1.train.Saver` checkpoints from TensorFlow 1.x can be loaded using this method. Names are used to match variables. Re-encode name-based checkpoints using `tf.train.Checkpoint.save` as soon as possible. **Loading from SavedModel checkpoints** To load values from a SavedModel, just pass the SavedModel directory to checkpoint.restore: ```python model = tf.keras.Model(...) tf.saved_model.save(model, path) # or model.save(path, save_format='tf') checkpoint = tf.train.Checkpoint(model) checkpoint.restore(path).expect_partial() ``` This example calls `expect_partial()` on the loaded status, since SavedModels saved from Keras often generates extra keys in the checkpoint. Otherwise, the program prints a lot of warnings about unused keys at exit time. Args: save_path: The path to the checkpoint, as returned by `save` or `tf.train.latest_checkpoint`. If the checkpoint was written by the name-based `tf.compat.v1.train.Saver`, names are used to match variables. This path may also be a SavedModel directory. options: Optional `tf.train.CheckpointOptions` object. Returns: A load status object, which can be used to make assertions about the status of a checkpoint restoration. The returned status object has the following methods: * `assert_consumed()`: Raises an exception if any variables are unmatched: either checkpointed values which don't have a matching Python object or Python objects in the dependency graph with no values in the checkpoint. This method returns the status object, and so may be chained with other assertions. * `assert_existing_objects_matched()`: Raises an exception if any existing Python objects in the dependency graph are unmatched. Unlike `assert_consumed`, this assertion will pass if values in the checkpoint have no corresponding Python objects. For example a `tf.keras.Layer` object which has not yet been built, and so has not created any variables, will pass this assertion but fail `assert_consumed`. Useful when loading part of a larger checkpoint into a new Python program, e.g. a training checkpoint with a `tf.compat.v1.train.Optimizer` was saved but only the state required for inference is being loaded. This method returns the status object, and so may be chained with other assertions. * `assert_nontrivial_match()`: Asserts that something aside from the root object was matched. This is a very weak assertion, but is useful for sanity checking in library code where objects may exist in the checkpoint which haven't been created in Python and some Python objects may not have a checkpointed value. * `expect_partial()`: Silence warnings about incomplete checkpoint restores. Warnings are otherwise printed for unused parts of the checkpoint file or object when the `Checkpoint` object is deleted (often at program shutdown). Raises: NotFoundError: if the a checkpoint or SavedModel cannot be found at `save_path`. Nrfz6Error when restoring from checkpoint or SavedModel at rz| Please double-check that the path is correct. You may be missing the checkpoint suffix (e.g. the '-1' in 'path/to/ckpt-1').)rrrrr IsDirectoryExistsrZget_saved_model_pb_pathZget_saved_model_pbtxt_pathZget_variables_pathrr rrrrmessagerrr8r)rBrrtZorig_save_pathr er.r.r/r s4j       zCheckpoint.restore)N)N)NF)N)N)N)r[r\r]r^rDrrrr`rrhrrrr.r.rr/rsdE - - f 1r)NN)N)or^rrFrr7rr|rrwrtensorflow.core.protobufrtensorflow.python.checkpointrrrrrrrrtensorflow.python.clientr Z session_libtensorflow.python.eagerr r r tensorflow.python.frameworkrrrrrrtensorflow.python.lib.iortensorflow.python.opsrrrrrrtensorflow.python.platformrrrhtensorflow.python.saved_modelr0tensorflow.python.saved_model.pywrap_saved_modelrtensorflow.python.trackablerr r!tensorflow.python.trainingr"r#rF!tensorflow.python.training.savingr$tensorflow.python.utilr%r&r'r(r) tensorflow.python.util.tf_exportr*r3rLockrrrr{r0r4r6r>objectr?rarqrrfloat32rrrrcontextmanagerrrrrrr.rJrSessionInterfacerKrRrrrurvZ AutoTrackablerrr.r.r.r/s                                           ,7V 1 % Z!  ;I a