From dbf07550d59cebd7a1b52d52bb77ff020aa73c87 Mon Sep 17 00:00:00 2001 From: Patrick McCarty Date: Mon, 24 Jun 2019 12:21:02 -0700 Subject: [PATCH] util: add open_auto helper function This helper function is meant to be used throughout autospec as a replacement for most uses of open(), especially whenever reading arbitrary data from upstream sources. It reads/writes data in UTF-8 always, with invalid UTF-8 characters escaped with Python's "surrogate" escaping mechanism. This ensures that any Latin-1 or other non-UTF-8 compatible encoded data can successfully be read in and later written out without data corruption. Signed-off-by: Patrick McCarty --- autospec/util.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/autospec/util.py b/autospec/util.py index b8f6222..836ae82 100644 --- a/autospec/util.py +++ b/autospec/util.py @@ -96,3 +96,16 @@ def write_out(filename, content, mode="w", encode=None): """File.write convenience wrapper.""" with open(filename, mode, encoding=encode) as require_f: require_f.write(content) + + +def open_auto(*args, **kwargs): + """ + Open a file with UTF-8 encoding, and "surrogate" escape characters that are + not valid UTF-8 to avoid data corruption. + """ + # 'encoding' and 'errors' are fourth and fifth positional arguments, so + # restrict the args tuple to (file, mode, buffering) at most + assert len(args) <= 3 + assert 'encoding' not in kwargs + assert 'errors' not in kwargs + return open(*args, encoding="utf-8", errors="surrogateescape", **kwargs)