EVOLUTION-MANAGER

Edit File: directory_watcher.py

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Contains the implementation for the DirectoryWatcher class."""

import bisect

from tensorboard.backend.event_processing import io_wrapper
from tensorboard.compat import tf
from tensorboard.util import tb_logging

logger = tb_logging.get_logger()

class DirectoryWatcher(object):
    """A DirectoryWatcher wraps a loader to load from a sequence of paths.

A loader reads a path and produces some kind of values as an iterator. A
    DirectoryWatcher takes a directory, a factory for loaders, and optionally a
    path filter and watches all the paths inside that directory.

This class is only valid under the assumption that only one path will be
    written to by the data source at a time and that once the source stops writing
    to a path, it will start writing to a new path that's lexicographically
    greater and never come back. It uses some heuristics to check whether this is
    true based on tracking changes to the files' sizes, but the check can have
    false negatives. However, it should have no false positives.
    """

def __init__(self, directory, loader_factory, path_filter=lambda x: True):
        """Constructs a new DirectoryWatcher.

Args:
          directory: The directory to load files from.
          loader_factory: A factory for creating loaders. The factory should take a
            path and return an object that has a Load method returning an
            iterator that will yield all events that have not been yielded yet.
          path_filter: If specified, only paths matching this filter are loaded.

Raises:
          ValueError: If path_provider or loader_factory are None.
        """
        if directory is None:
            raise ValueError("A directory is required")
        if loader_factory is None:
            raise ValueError("A loader factory is required")
        self._directory = directory
        self._path = None
        self._loader_factory = loader_factory
        self._loader = None
        self._path_filter = path_filter
        self._ooo_writes_detected = False
        # The file size for each file at the time it was finalized.
        self._finalized_sizes = {}

def Load(self):
        """Loads new values.

The watcher will load from one path at a time; as soon as that path stops
        yielding events, it will move on to the next path. We assume that old paths
        are never modified after a newer path has been written. As a result, Load()
        can be called multiple times in a row without losing events that have not
        been yielded yet. In other words, we guarantee that every event will be
        yielded exactly once.

Yields:
          All values that have not been yielded yet.

Raises:
          DirectoryDeletedError: If the directory has been permanently deleted
            (as opposed to being temporarily unavailable).
        """
        try:
            for event in self._LoadInternal():
                yield event
        except tf.errors.OpError:
            if not tf.io.gfile.exists(self._directory):
                raise DirectoryDeletedError(
                    "Directory %s has been permanently deleted"
                    % self._directory
                )

def _LoadInternal(self):
        """Internal implementation of Load().

The only difference between this and Load() is that the latter will throw
        DirectoryDeletedError on I/O errors if it thinks that the directory has been
        permanently deleted.

Yields:
          All values that have not been yielded yet.
        """

# If the loader exists, check it for a value.
        if not self._loader:
            self._InitializeLoader()

# If it still doesn't exist, there is no data
        if not self._loader:
            return

while True:
            # Yield all the new events in the path we're currently loading from.
            for event in self._loader.Load():
                yield event

next_path = self._GetNextPath()
            if not next_path:
                logger.info("No path found after %s", self._path)
                # Current path is empty and there are no new paths, so we're done.
                return

# There's a new path, so check to make sure there weren't any events
            # written between when we finished reading the current path and when we
            # checked for the new one. The sequence of events might look something
            # like this:
            #
            # 1. Event #1 written to path #1.
            # 2. We check for events and yield event #1 from path #1
            # 3. We check for events and see that there are no more events in path #1.
            # 4. Event #2 is written to path #1.
            # 5. Event #3 is written to path #2.
            # 6. We check for a new path and see that path #2 exists.
            #
            # Without this loop, we would miss event #2. We're also guaranteed by the
            # loader contract that no more events will be written to path #1 after
            # events start being written to path #2, so we don't have to worry about
            # that.
            for event in self._loader.Load():
                yield event

logger.info(
                "Directory watcher advancing from %s to %s",
                self._path,
                next_path,
            )

# Advance to the next path and start over.
            self._SetPath(next_path)

# The number of paths before the current one to check for out of order writes.
    _OOO_WRITE_CHECK_COUNT = 20

def OutOfOrderWritesDetected(self):
        """Returns whether any out-of-order writes have been detected.

Out-of-order writes are only checked as part of the Load() iterator. Once an
        out-of-order write is detected, this function will always return true.

Note that out-of-order write detection is not performed on GCS paths, so
        this function will always return false.

Returns:
          Whether any out-of-order write has ever been detected by this watcher.
        """
        return self._ooo_writes_detected

def _InitializeLoader(self):
        path = self._GetNextPath()
        if path:
            self._SetPath(path)

def _SetPath(self, path):
        """Sets the current path to watch for new events.

This also records the size of the old path, if any. If the size can't be
        found, an error is logged.

Args:
          path: The full path of the file to watch.
        """
        old_path = self._path
        if old_path and not io_wrapper.IsCloudPath(old_path):
            try:
                # We're done with the path, so store its size.
                size = tf.io.gfile.stat(old_path).length
                logger.debug("Setting latest size of %s to %d", old_path, size)
                self._finalized_sizes[old_path] = size
            except tf.errors.OpError as e:
                logger.error("Unable to get size of %s: %s", old_path, e)

self._path = path
        self._loader = self._loader_factory(path)

def _GetNextPath(self):
        """Gets the next path to load from.

This function also does the checking for out-of-order writes as it iterates
        through the paths.

Returns:
          The next path to load events from, or None if there are no more paths.
        """
        paths = sorted(
            path
            for path in io_wrapper.ListDirectoryAbsolute(self._directory)
            if self._path_filter(path)
        )
        if not paths:
            return None

if self._path is None:
            return paths[0]

# Don't bother checking if the paths are GCS (which we can't check) or if
        # we've already detected an OOO write.
        if (
            not io_wrapper.IsCloudPath(paths[0])
            and not self._ooo_writes_detected
        ):
            # Check the previous _OOO_WRITE_CHECK_COUNT paths for out of order writes.
            current_path_index = bisect.bisect_left(paths, self._path)
            ooo_check_start = max(
                0, current_path_index - self._OOO_WRITE_CHECK_COUNT
            )
            for path in paths[ooo_check_start:current_path_index]:
                if self._HasOOOWrite(path):
                    self._ooo_writes_detected = True
                    break

next_paths = list(
            path for path in paths if self._path is None or path > self._path
        )
        if next_paths:
            return min(next_paths)
        else:
            return None

def _HasOOOWrite(self, path):
        """Returns whether the path has had an out-of-order write."""
        # Check the sizes of each path before the current one.
        size = tf.io.gfile.stat(path).length
        old_size = self._finalized_sizes.get(path, None)
        if size != old_size:
            if old_size is None:
                logger.error(
                    "File %s created after file %s even though it's "
                    "lexicographically earlier",
                    path,
                    self._path,
                )
            else:
                logger.error(
                    "File %s updated even though the current file is %s",
                    path,
                    self._path,
                )
            return True
        else:
            return False

class DirectoryDeletedError(Exception):
    """Thrown by Load() when the directory is *permanently* gone.

We distinguish this from temporary errors so that other code can
    decide to drop all of our data only when a directory has been
    intentionally deleted, as opposed to due to transient filesystem
    errors.
    """

pass