#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
SavReaderWriter.py: A cross-platform Python interface to the IBM SPSS
Statistics Input Output Module. Read or Write SPSS system files (.sav, .zsav)
"""
# libspssdio.so.1, libspssdio.dylib, spssio32.dll + associated libaries and
# documentation can be downloaded here:
# https://www.ibm.com/developerworks/mydeveloperworks/wikis/home/wiki/We70df3195ec8_4f95_9773_42e448fa9029/page/Downloads%20for%20IBM%C2%AE%20SPSS%C2%AE%20Statistics?lang=en
# ANY FEEDBACK ON THIS CODE IS WELCOME: "@".join(["fomcl", "yahoo.com"])
# Mac support added Oct-22-2011: by Rich Sadowsky
# "@".join(["rich", "richsad.com"])
## January 2013:
# changed implementation of freeMemory method (again!)
## December 2012:
# -Added support for slicing, indexing, array slicing + other special methods
# -Added support for writing spss date fields
# -Added support for almost all meta data (missing values, sets, roles, etc.)
# -Added support for 64 bit Windows (tested with Win7) and other OSs
# (z/Linux, Solaris, HP Linux, IBM AIX (untested though)
# -Added support for reading and writing zlib compressed (.zsav) files
# -Removed pesky segfault error when freeing memory
# -Removed errors related to encoding
# -Removed error that prevented files originating from big-endian systems
# being read properly in little-endian systems (and vice versa)
# -Changed some Reader defaults (verbose=False, returnHeader=False)
# -Renamed SavDataDictionaryReader into SavHeaderReader
##
__author__ = "Albert-Jan Roskam" + " " + "@".join(["fomcl", "yahoo.com"])
__version__ = "3.1.1"
# change this to 'True' in case you experience segmentation
# faults related to freeing memory.
segfaults = False
from ctypes import *
import ctypes.util
import struct
import sys
import platform
import os
import re
import operator
import math
import locale
import datetime
import time
import getpass
import encodings
import functools
import gc
try:
import psyco
psycoOk = True # reading 66 % faster
except ImportError:
print ("NOTE. Psyco module not found. Install this module " +
"to increase reader performance")
psycoOk = False
try:
import numpy
numpyOk = True
except ImportError:
print ("NOTE. Numpy module not found. Install this module " +
"to use array slicing")
numpyOk = False
try:
from cWriterow import cWriterow # writing 66 % faster
cWriterowOK = True
except ImportError:
print ("NOTE. cWriterow module not found. Install this module " +
"to increase writer performance")
cWriterowOK = False
retcodes = {
0: "SPSS_OK",
1: "SPSS_FILE_OERROR",
2: "SPSS_FILE_WERROR",
3: "SPSS_FILE_RERROR",
4: "SPSS_FITAB_FULL",
5: "SPSS_INVALID_HANDLE",
6: "SPSS_INVALID_FILE",
7: "SPSS_NO_MEMORY",
8: "SPSS_OPEN_RDMODE",
9: "SPSS_OPEN_WRMODE",
10: "SPSS_INVALID_VARNAME",
11: "SPSS_DICT_EMPTY",
12: "SPSS_VAR_NOTFOUND",
13: "SPSS_DUP_VAR",
14: "SPSS_NUME_EXP",
15: "SPSS_STR_EXP",
16: "SPSS_SHORTSTR_EXP",
17: "SPSS_INVALID_VARTYPE",
18: "SPSS_INVALID_MISSFOR",
19: "SPSS_INVALID_COMPSW",
20: "SPSS_INVALID_PRFOR",
21: "SPSS_INVALID_WRFOR",
22: "SPSS_INVALID_DATE",
23: "SPSS_INVALID_TIME",
24: "SPSS_NO_VARIABLES",
27: "SPSS_DUP_VALUE",
28: "SPSS_INVALID_CASEWGT",
30: "SPSS_DICT_COMMIT",
31: "SPSS_DICT_NOTCOMMIT",
33: "SPSS_NO_TYPE2",
41: "SPSS_NO_TYPE73",
45: "SPSS_INVALID_DATEINFO",
46: "SPSS_NO_TYPE999",
47: "SPSS_EXC_STRVALUE",
48: "SPSS_CANNOT_FREE",
49: "SPSS_BUFFER_SHORT",
50: "SPSS_INVALID_CASE",
51: "SPSS_INTERNAL_VLABS",
52: "SPSS_INCOMPAT_APPEND",
53: "SPSS_INTERNAL_D_A",
54: "SPSS_FILE_BADTEMP",
55: "SPSS_DEW_NOFIRST",
56: "SPSS_INVALID_MEASURELEVEL",
57: "SPSS_INVALID_7SUBTYPE",
58: "SPSS_INVALID_VARHANDLE",
59: "SPSS_INVALID_ENCODING",
60: "SPSS_FILES_OPEN",
70: "SPSS_INVALID_MRSETDEF",
71: "SPSS_INVALID_MRSETNAME",
72: "SPSS_DUP_MRSETNAME",
73: "SPSS_BAD_EXTENSION",
74: "SPSS_INVALID_EXTENDEDSTRING",
75: "SPSS_INVALID_ATTRNAME",
76: "SPSS_INVALID_ATTRDEF",
77: "SPSS_INVALID_MRSETINDEX",
78: "SPSS_INVALID_VARSETDEF",
79: "SPSS_INVALID_ROLE",
-15: "SPSS_EMPTY_DEW",
-14: "SPSS_NO_DEW",
-13: "SPSS_EMPTY_MULTRESP",
-12: "SPSS_NO_MULTRESP",
-11: "SPSS_NO_DATEINFO",
-10: "SPSS_NO_CASEWGT",
-9: "SPSS_NO_LABEL",
-8: "SPSS_NO_LABELS",
-7: "SPSS_EMPTY_VARSETS",
-6: "SPSS_NO_VARSETS",
-5: "SPSS_FILE_END",
-4: "SPSS_EXC_VALLABEL",
-3: "SPSS_EXC_LEN120",
-2: "SPSS_EXC_VARLABEL",
-1: "SPSS_EXC_LEN64"}
allFormats = {
1: ("SPSS_FMT_A", "Alphanumeric"),
2: ("SPSS_FMT_AHEX", "Alphanumeric hexadecimal"),
3: ("SPSS_FMT_COMMA", "F Format with commas"),
4: ("SPSS_FMT_DOLLAR", "Commas and floating dollar sign"),
5: ("SPSS_FMT_F", "Default Numeric Format"),
6: ("SPSS_FMT_IB", "Integer binary"),
7: ("SPSS_FMT_PIBHEX", "Positive integer binary - hex"),
8: ("SPSS_FMT_P", "Packed decimal"),
9: ("SPSS_FMT_PIB", "Positive integer binary unsigned"),
10: ("SPSS_FMT_PK", "Positive integer binary unsigned"),
11: ("SPSS_FMT_RB", "Floating point binary"),
12: ("SPSS_FMT_RBHEX", "Floating point binary hex"),
15: ("SPSS_FMT_Z", "Zoned decimal"),
16: ("SPSS_FMT_N", "N Format- unsigned with leading 0s"),
17: ("SPSS_FMT_E", "E Format- with explicit power of 10"),
20: ("SPSS_FMT_DATE", "Date format dd-mmm-yyyy"),
21: ("SPSS_FMT_TIME", "Time format hh:mm:ss.s"),
22: ("SPSS_FMT_DATE_TIME", "Date and Time"),
23: ("SPSS_FMT_ADATE", "Date format dd-mmm-yyyy"),
24: ("SPSS_FMT_JDATE", "Julian date - yyyyddd"),
25: ("SPSS_FMT_DTIME", "Date-time dd hh:mm:ss.s"),
26: ("SPSS_FMT_WKDAY", "Day of the week"),
27: ("SPSS_FMT_MONTH", "Month"),
28: ("SPSS_FMT_MOYR", "mmm yyyy"),
29: ("SPSS_FMT_QYR", "q Q yyyy"),
30: ("SPSS_FMT_WKYR", "ww WK yyyy"),
31: ("SPSS_FMT_PCT", "Percent - F followed by %"),
32: ("SPSS_FMT_DOT", "Like COMMA, switching dot for comma"),
33: ("SPSS_FMT_CCA", "User Programmable currency format"),
34: ("SPSS_FMT_CCB", "User Programmable currency format"),
35: ("SPSS_FMT_CCC", "User Programmable currency format"),
36: ("SPSS_FMT_CCD", "User Programmable currency format"),
37: ("SPSS_FMT_CCE", "User Programmable currency format"),
38: ("SPSS_FMT_EDATE", "Date in dd/mm/yyyy style"),
39: ("SPSS_FMT_SDATE", "Date in yyyy/mm/dd style")}
MAXLENGTHS = {
"SPSS_MAX_VARNAME": (64, "Variable name"),
"SPSS_MAX_SHORTVARNAME": (8, "Short (compatibility) variable name"),
"SPSS_MAX_SHORTSTRING": (8, "Short string variable"),
"SPSS_MAX_IDSTRING": (64, "File label string"),
"SPSS_MAX_LONGSTRING": (32767, "Long string variable"),
"SPSS_MAX_VALLABEL": (120, "Value label"),
"SPSS_MAX_VARLABEL": (256, "Variable label"),
"SPSS_MAX_7SUBTYPE": (40, "Maximum record 7 subtype"),
"SPSS_MAX_ENCODING": (64, "Maximum encoding text")}
supportedDates = { # uses ISO dates wherever applicable.
"DATE": "%Y-%m-%d",
"JDATE": "%Y-%m-%d",
"EDATE": "%Y-%m-%d",
"SDATE": "%Y-%m-%d",
"DATETIME": "%Y-%m-%d %H:%M:%S",
"WKDAY": "%A %H:%M:%S",
"ADATE": "%Y-%m-%d",
"WKDAY": "%A",
"MONTH": "%B",
"MOYR": "%B %Y",
"WKYR": "%W WK %Y"}
userMissingValues = {
"SPSS_NO_MISSVAL": 0,
"SPSS_ONE_MISSVAL": 1,
"SPSS_TWO_MISSVAL": 2,
"SPSS_THREE_MISSVAL": 3,
"SPSS_MISS_RANGE": -2,
"SPSS_MISS_RANGEANDVAL": -3}
class SPSSIOError(Exception):
"""
Error class for the IBM SPSS Statistics Input Output Module
"""
def __init__(self, msg=u"Unknown", retcode=None):
if isinstance(retcode, int):
code = retcodes.get(retcode, retcode)
elif isinstance(retcode, list):
code = ", ".join([retcodes.get(retcodex) for retcodex in retcode])
else:
code = "unknown code %r" % retcode
msg = "%s [retcode: %r]" % (msg, code)
Exception.__init__(self, msg)
class Generic(object):
"""
Class for methods and data used in reading as well as writing
IBM SPSS Statistics data files
"""
def __init__(self, savFileName, ioUtf8=False, ioLocale=None):
"""Constructor. Note that interface locale and encoding can only
be set once"""
locale.setlocale(locale.LC_ALL, "")
self.savFileName = savFileName
self.libc = cdll.LoadLibrary(ctypes.util.find_library("c"))
self.spssio = self.loadLibrary()
self.wholeCaseIn = self.spssio.spssWholeCaseIn
self.wholeCaseOut = self.spssio.spssWholeCaseOut
self.encoding_and_locale_set = False
if not self.encoding_and_locale_set:
self.encoding_and_locale_set = True
self.ioLocale = ioLocale
self.ioUtf8 = ioUtf8
def _encodeFileName(self, fn):
"""Helper function to encode unicode file names into bytestring file
names encoded in the file system's encoding. Needed for C functions
that have a c_char_p filename argument.
http://effbot.org/pyref/sys.getfilesystemencoding.htm
http://docs.python.org/2/howto/unicode.html under 'unicode filenames'"""
if not isinstance(fn, unicode):
return fn
elif sys.platform.startswith("win"):
return self.wide2utf8(fn)
else:
encoding = sys.getfilesystemencoding()
encoding = "utf-8" if not encoding else encoding # actually, ascii
try:
return fn.encode(encoding)
except UnicodeEncodeError, e:
msg = ("File system encoding %r can not be used to " +
"encode file name %r [%s]")
raise ValueError(msg % (encoding, fn, e))
def loadLibrary(self):
"""This function loads and returns the SPSSIO libraries,
depending on the platform."""
path = os.path.abspath(".")
is_32bit = platform.architecture()[0] == "32bit"
pf = sys.platform.lower()
load = WinDLL if pf.startswith("win") else CDLL
if pf.startswith("win"):
os.environ["PATH"] += ";" + path
spssio = load("spssio32") if is_32bit else load("spssio64")
elif pf.startswith("lin"): # most linux flavours, incl zLinux
spssio = load("libspssdio.so.1")
elif pf.startswith("darwin") or pf.startswith("mac"):
spssio = load("libspssdio.dylib")
elif pf.startswith("aix") and not is_32bit:
spssio = load("libspssdio.so.1")
elif pf.startswith("hp-ux"):
spssio = load("libspssdio.sl.1")
elif pf.startswith("sunos") and not is_32bit:
spssio = load("libspssdio.so.1")
else:
msg = "Your platform (%r) is not supported" % pf
raise NotImplementedError(msg)
return spssio
def errcheck(self, res, func, args):
"""This function checks for errors during the execution of
function <func>"""
if not res:
msg = "Error performing %r operation on file %r."
raise IOError(msg % (func.__name__, self.savFileName))
return res
def wide2utf8(self, fn):
"""Take a unicode file name string and encode it to a multibyte string
that Windows can use to represent file names (CP65001, UTF-8)
http://msdn.microsoft.com/en-us/library/windows/desktop/dd374130"""
from ctypes import wintypes
_CP_UTF8 = 65001
_CP_ACP = 0 # ANSI
_LPBOOL = POINTER(c_long)
_wideCharToMultiByte = windll.kernel32.WideCharToMultiByte
_wideCharToMultiByte.restype = c_int
_wideCharToMultiByte.argtypes = [wintypes.UINT, wintypes.DWORD,
wintypes.LPCWSTR, c_int, wintypes.LPSTR, c_int, wintypes.LPCSTR, _LPBOOL]
codePage = _CP_UTF8
dwFlags = 0
lpWideCharStr = fn
cchWideChar = len(fn)
lpMultiByteStr = None
cbMultiByte = 0 # zero requests size
lpDefaultChar = None
lpUsedDefaultChar = None
# get size
mbcssize = _wideCharToMultiByte(
codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
cbMultiByte, lpDefaultChar, lpUsedDefaultChar)
if mbcssize <= 0:
raise WinError(mbcssize)
lpMultiByteStr = create_string_buffer(mbcssize)
# convert
retcode = _wideCharToMultiByte(
codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
mbcssize, lpDefaultChar, lpUsedDefaultChar)
if retcode <= 0:
raise WinError(retcode)
return lpMultiByteStr.value
def openSavFile(self, savFileName, mode="rb", refSavFileName=None):
"""This function opens IBM SPSS Statistics data files in mode <mode>
and returns a handle that should be used for subsequent operations on
the file. If <savFileName> is opened in mode "cp", meta data
information aka the spss dictionary is copied from <refSavFileName>"""
savFileName = os.path.abspath(savFileName) # fdopen wants full name
try:
fdopen = self.libc._fdopen # Windows
except AttributeError:
fdopen = self.libc.fdopen # Linux and others
fdopen.argtypes = [c_int, c_char_p]
fdopen.restype = c_void_p
fdopen.errcheck = self.errcheck
with open(savFileName, mode) as f:
self.fd = fdopen(f.fileno(), mode)
if mode == "rb":
spssOpen = self.spssio.spssOpenRead
elif mode == "wb":
spssOpen = self.spssio.spssOpenWrite
elif mode == "cp":
spssOpen = self.spssio.spssOpenWriteCopy
elif mode == "ab":
spssOpen = self.spssio.spssOpenAppend
savFileName = self._encodeFileName(savFileName)
refSavFileName = self._encodeFileName(refSavFileName)
sav = c_char_p(savFileName)
fh = c_int(self.fd)
if mode == "cp":
retcode = spssOpen(sav, c_char_p(refSavFileName), pointer(fh))
else:
retcode = spssOpen(sav, pointer(fh))
if retcode > 0:
msg = "Error opening file %r in mode %r"
raise SPSSIOError(msg % (savFileName, mode), retcode)
return fh.value
def closeSavFile(self, fh, mode="rb"):
"""This function closes the sav file associated with <fh> that was open
in mode <mode>."""
if mode == "rb":
spssClose = self.spssio.spssCloseRead
elif mode == "wb":
spssClose = self.spssio.spssCloseWrite
elif mode == "ab":
spssClose = self.spssio.spssCloseAppend
retcode = spssClose(c_int(fh))
if retcode > 0:
raise SPSSIOError("Error closing file in mode %r" % mode, retcode)
@property
def releaseInfo(self):
"""This function reports release- and machine-specific information
about the open file."""
relInfo = ["release number", "release subnumber", "fixpack number",
"machine code", "floating-point representation code",
"compression scheme code", "big/little-endian code",
"character representation code"]
relInfoArr = (c_int * len(relInfo))()
retcode = self.spssio.spssGetReleaseInfo(c_int(self.fh), relInfoArr)
if retcode > 0:
raise SPSSIOError("Error getting ReleaseInfo", retcode)
info = dict([(item, relInfoArr[i]) for i, item in enumerate(relInfo)])
return info
@property
def spssVersion(self):
"""Return the SPSS version that was used to create the opened file
as a three-tuple indicating major, minor, and fixpack version as
ints. NB: in the transition from SPSS to IBM, a new four-digit
versioning nomenclature is used. This function returns the old
three-digit nomenclature. Therefore, no patch version information
is available."""
info = self.releaseInfo
major = info["release number"]
minor = info["release subnumber"]
fixpack = info["fixpack number"]
return major, minor, fixpack
@property
def fileCompression(self):
"""Get/Set the file compression.
Returns/Takes a compression switch which may be any of the following:
'uncompressed', 'standard', or 'zlib'. Zlib comression requires SPSS
v21 I/O files."""
compression = {0: "uncompressed", 1: "standard", 2: "zlib"}
compSwitch = c_int()
func = self.spssio.spssGetCompression
retcode = func(c_int(self.fh), byref(compSwitch))
if retcode > 0:
raise SPSSIOError("Error getting file compression", retcode)
return compression.get(compSwitch.value)
@fileCompression.setter
def fileCompression(self, compSwitch):
compression = {"uncompressed": 0, "standard": 1, "zlib": 2}
compSwitch = compression.get(compSwitch)
func = self.spssio.spssSetCompression
retcode = func(c_int(self.fh), c_int(compSwitch))
invalidSwitch = retcodes.get(retcode) == 'SPSS_INVALID_COMPSW'
if invalidSwitch and self.spssVersion[0] < 21:
msg = "Writing zcompressed files requires >=v21 SPSS I/O libraries"
raise ValueError(msg)
elif retcode > 0:
raise SPSSIOError("Error setting file compression", retcode)
@property
def systemString(self):
"""This function returns the name of the system under which the file
was created aa a string."""
sysName = create_string_buffer(42)
func = self.spssio.spssGetSystemString
retcode = func(c_int(self.fh), byref(sysName))
if retcode > 0:
raise SPSSIOError("Error getting SystemString", retcode)
return sysName.value
def getStruct(self, varTypes, varNames, mode="rb"):
"""This function returns a compiled struct object. The required
struct format string for the conversion between C and Python
is created on the basis of varType and byte order.
--varTypes: SPSS data files have either 8-byte doubles/floats or n-byte
chars[]/ strings, where n is always 8 bytes or a multiple thereof.
--byte order: files are written in the byte order of the host system
(mode="wb") and read/appended using the byte order information
contained in the SPSS data file (mode is "ab" or "rb" or "cp")"""
if mode in ("ab", "rb", "cp"): # derive endianness from file
endianness = self.releaseInfo["big/little-endian code"]
endianness = ">" if endianness > 0 else "<"
elif mode == "wb": # derive endianness from host
if sys.byteorder == "little":
endianness = "<"
elif sys.byteorder == "big":
endianness = ">"
else:
endianness = "@"
structFmt = [endianness]
ceil = math.ceil
for varName in varNames:
varType = varTypes[varName]
if varType == 0:
structFmt.append("d")
else:
fmt = str(int(ceil(int(varType) / 8.0) * 8))
structFmt.append(fmt + "s")
return struct.Struct("".join(structFmt))
def getCaseBuffer(self):
"""This function returns a buffer and a pointer to that buffer. A whole
case will be read into this buffer."""
caseSize = c_long()
retcode = self.spssio.spssGetCaseSize(c_int(self.fh), byref(caseSize))
caseBuffer = create_string_buffer(caseSize.value)
if retcode > 0:
raise SPSSIOError("Problem getting case buffer", retcode)
return caseBuffer
@property
def sysmis(self):
"""This function returns the IBM SPSS Statistics system-missing
value ($SYSMIS) for the host system (also called 'NA' in other
systems)."""
try:
sysmis = -1 * sys.float_info[0] # Python 2.6 and higher.
except AttributeError:
self.spssio.spssSysmisVal.restype = c_float
sysmis = self.spssio.spssSysmisVal()
return sysmis
@property
def missingValuesLowHigh(self):
"""This function returns the 'lowest' and 'highest' values used for
numeric missing value ranges on the host system. This can be used in
a similar way as the LO and HI keywords in missing values
specifications (cf. MISSING VALUES foo (LO THRU 0). It may be called
at any time."""
lowest, highest = c_double(), c_double()
func = self.spssio.spssLowHighVal
retcode = func(byref(lowest), byref(highest))
return lowest.value, highest.value
@property
def ioLocale(self):
"""This function gets/sets the I/O Module's locale.
This corresponds with the SPSS command SET LOCALE. The I/O Module's
locale is separate from that of the client application. The
<localeName> parameter and the return value are identical to those
for the C run-time function setlocale. The exact locale name
specification depends on the OS of the host sytem, but has the
following form:
<lang>_<territory>.<codeset>[@<modifiers>]
The 'codeset' and 'modifier' components are optional and in Windows,
aliases (e.g. 'english') may be used. When the I/O Module is first
loaded, its locale is set to the system default. See also:
--https://wiki.archlinux.org/index.php/Locale
--http://msdn.microsoft.com/en-us/library/39cwe7zf(v=vs.80).aspx"""
if hasattr(self, "setLocale"):
return self.setLocale
else:
currLocale = ".".join(locale.getlocale())
print "NOTE. Locale not set; getting current locale: ", currLocale
return currLocale
@ioLocale.setter
def ioLocale(self, localeName=""):
if not localeName:
localeName = ".".join(locale.getlocale())
func = self.spssio.spssSetLocale
func.restype = c_char_p
self.setLocale = func(c_int(locale.LC_ALL), c_char_p(localeName))
if self.setLocale is None:
raise ValueError("Invalid ioLocale: %r" % localeName)
return self.setLocale
@property
def fileCodePage(self):
"""This function provides the Windows code page number of the encoding
applicable to a file."""
nCodePage = c_int()
func = self.spssio.spssGetFileCodePage
retcode = func(c_int(self.fh), byref(nCodePage))
return nCodePage.value
def isCompatibleEncoding(self):
"""This function determines whether the file and interface encoding
are compatible."""
try:
# Windows, note typo 'Endoding'!
func = self.spssio.spssIsCompatibleEndoding
except AttributeError:
func = self.spssio.spssIsCompatibleEncoding
func.restype = c_bool
isCompatible = c_int()
retcode = func(c_int(self.fh), byref(isCompatible))
if retcode > 0:
msg = "Error testing encoding compatibility: %r"
raise SPSSIOError(msg % isCompatible.value, retcode)
if not isCompatible.value and not self.ioUtf8:
msg = ("NOTE. SPSS Statistics data file %r is written in a " +
"character encoding (%s) incompatible with the current " +
"ioLocale setting. It may not be readable. Consider " +
"changing ioLocale or setting ioUtf8=True.")
print msg % (self.savFileName, self.fileEncoding)
return bool(isCompatible.value)
@property
def ioUtf8(self):
"""This function returns/sets the current interface encoding.
ioUtf8 = False --> CODEPAGE mode,
ioUtf8 = True --> UTF-8 mode, aka. Unicode mode
This corresponds with the SPSS command SHOW UNICODE (getter)
and SET UNICODE=ON/OFF (setter)."""
if hasattr(self, "ioUtf8_"):
return self.ioUtf8_
self.ioUtf8_ = self.spssio.spssGetInterfaceEncoding()
return bool(self.ioUtf8_)
@ioUtf8.setter
def ioUtf8(self, ioUtf8):
try:
retcode = self.spssio.spssSetInterfaceEncoding(c_int(int(ioUtf8)))
if retcode > 0 and not self.encoding_and_locale_set:
# not self.encoding_and_locale_set --> nested context managers
raise SPSSIOError("Error setting IO interface", retcode)
except TypeError:
msg = "Invalid interface encoding: %r (must be bool)"
raise SPSSIOError(msg % ioUtf8)
@property
def fileEncoding(self):
"""This function obtains the encoding applicable to a file.
The encoding is returned as an IANA encoding name, such as
ISO-8859-1, which is then converted to the corresponding Python
codec name. If the file contains no file encoding, the locale's
preferred encoding is returned"""
preferredEncoding = locale.getpreferredencoding()
try:
pszEncoding = create_string_buffer(20) # is 20 enough??
func = self.spssio.spssGetFileEncoding
retcode = func(c_int(self.fh), byref(pszEncoding))
if retcode > 0:
raise SPSSIOError("Error getting file encoding", retcode)
iana_codes = encodings.aliases.aliases
rawEncoding = pszEncoding.value.lower()
if rawEncoding.replace("-", "") in iana_codes:
iana_code = rawEncoding.replace("-", "")
else:
iana_code = rawEncoding.replace("-", "_")
fileEncoding = iana_codes[iana_code]
return fileEncoding
except AttributeError:
print ("NOTE. Function 'getFileEncoding' not found. You are " +
"using a .dll from SPSS < v16.")
return preferredEncoding
except KeyError:
print ("NOTE. IANA coding lookup error. Code %r " % iana_code +
"does not map to any Python codec.")
return preferredEncoding
@property
def record(self):
"""Get/Set a whole record from/to a pre-allocated buffer"""
retcode = self.wholeCaseIn(c_int(self.fh),
byref(self.caseBuffer))
if retcode > 0:
raise SPSSIOError("Problem reading row", retcode)
record = list(self.unpack_from(self.caseBuffer))
return record
@record.setter
def record(self, record):
try:
self.pack_into(self.caseBuffer, 0, *record)
except struct.error, e:
msg = "Use ioUtf8=True to write unicode strings [%s]" % e
raise TypeError(msg)
retcode = self.wholeCaseOut(c_int(self.fh),
c_char_p(self.caseBuffer.raw))
if retcode > 0:
raise SPSSIOError("Problem writing row:\n" + \
unicode(record, "utf-8"), retcode)
def printPctProgress(self, nominator, denominator):
"""This function prints the % progress when reading and writing
files"""
if nominator and nominator % 10**4 == 0:
pctProgress = (float(nominator) / denominator) * 100
print "%2.1f%%... " % pctProgress,
class Header(Generic):
"""
This class contains methods responsible for getting and setting meta data
that is embedded in the IBM SPSS Statistics data file. In SPSS speak, this
header information is known as the SPSS Data Dictionary (which has diddly
squat to do with a Python dictionary!).
"""
def __init__(self, savFileName, mode, refSavFileName, ioUtf8, ioLocale=None):
"""Constructor"""
super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
self.spssio = self.loadLibrary()
self.libc = cdll.LoadLibrary(ctypes.util.find_library("c"))
self.fh = super(Header, self).openSavFile(savFileName, mode,
refSavFileName)
self.varNames, self.varTypes = self.varNamesTypes
self.vNames = dict(zip(self.varNames, self.encode(self.varNames)))
def openSavFile(self):
"""This function returns the file handle that was opened in the
super class"""
return self.fh
def decode(func):
"""Decorator to Utf-8 decode all str items contained in a dictionary
If ioUtf8=True, the dictionary's keys and values are decoded, but only
values that are strs, lists, or dicts. For example:
{'v1': {'y': 'yy', 'z': 666}}-->{u'v1': {u'y': u'yy', u'z': 666}}"""
uS = lambda x: x.decode("utf-8") if isinstance(x, str) else x
uL = lambda x: map(uS, x) if isinstance(x, list) else x
@functools.wraps(func)
def wrapper(arg):
result = func(arg)
if not arg.ioUtf8:
return result # unchanged
if isinstance(result, str):
return uS(result)
utf8ifiedDict = {}
for k, v in result.iteritems():
k, v = uS(k), uS(v)
if isinstance(v, list):
v = map(uS, v)
elif isinstance(v, dict):
v = dict([(uS(x), uL(uS(y))) for x, y in v.items()])
utf8ifiedDict[k] = v
return utf8ifiedDict
return wrapper
def encode(self, item):
"""Counter part of decode helper function, does the opposite of that
function (but is not a decorator)"""
if not self.ioUtf8:
return item # unchanged
utf8dify = lambda x: x.encode("utf-8") if isinstance(x, unicode) else x
if isinstance(item, list):
return map(utf8dify, item)
elif isinstance(item, dict):
return dict([(utf8dify(x), utf8dify(y)) for x, y in item.items()])
return utf8dify(item)
def freeMemory(self, funcName, *args):
"""Clean up: free memory claimed by e.g. getValueLabels and
getVarNamesTypes"""
gc.collect()
if segfaults:
return
print "... freeing" , funcName[8:]
func = getattr(self.spssio, funcName)
retcode = func(*args)
if retcode > 0:
msg = "Error freeing memory using %s" % funcName
raise SPSSIOError(msg, retcode)
@property
def numberofCases(self):
"""This function reports the number of cases present in a data file.
Prehistoric files (< SPSS v6.0) don't contain nCases info, therefore
a guesstimate of the number of cases is given for those files"""
nCases = c_long()
func = self.spssio.spssGetNumberofCases
retcode = func(c_int(self.fh), byref(nCases))
if nCases.value == -1:
func = self.spssio.spssGetEstimatedNofCases
retcode = func(c_int(self.fh), byref(nCases))
if retcode > 0:
raise SPSSIOError("Error getting number of cases", retcode)
return nCases.value
@property
def numberofVariables(self):
"""This function returns the number of variables (columns) in the
spss dataset"""
numVars = c_int()
func = self.spssio.spssGetNumberofVariables
retcode = func(c_int(self.fh), byref(numVars))
if retcode > 0:
raise SPSSIOError("Error getting number of variables", retcode)
return numVars.value
@property
def varNamesTypes(self):
"""Get/Set variable names and types.
--Variable names is a list of the form ['var1', var2', 'etc']
--Variable types is a dictionary of the form {varName: varType}
The variable type code is an integer in the range 0-32767, 0
indicating a numeric variable (F8.2) and a positive value
indicating a string variable of that size (in bytes)."""
if hasattr(self, "varNames"):
return self.varNames, self.varTypes
# initialize arrays
numVars = self.numberofVariables
numVars_ = c_int(numVars)
varNamesArr = (POINTER(c_char_p * numVars))()
varTypesArr = (POINTER(c_int * numVars))()
# get variable names
func = self.spssio.spssGetVarNames
retcode = func(c_int(self.fh), byref(numVars_),
byref(varNamesArr), byref(varTypesArr))
if retcode > 0:
raise SPSSIOError("Error getting variable names & types", retcode)
# get array contents
varNames = [varNamesArr[0][i] for i in xrange(numVars)]
varTypes = [varTypesArr[0][i] for i in xrange(numVars)]
if self.ioUtf8:
varNames = [varName.decode("utf-8") for varName in varNames]
# clean up
args = (varNamesArr, varTypesArr, numVars)
self.freeMemory("spssFreeVarNames", *args)
return varNames, dict(zip(varNames, varTypes))
@varNamesTypes.setter
def varNamesTypes(self, varNamesVarTypes):
badLengthMsg = "Empty or longer than %s chars" % \
(MAXLENGTHS['SPSS_MAX_VARNAME'][0])
varNames, varTypes = varNamesVarTypes
varNameRetcodes = {
0: ('SPSS_NAME_OK', 'Valid standard name'),
1: ('SPSS_NAME_SCRATCH', 'Valid scratch var name'),
2: ('SPSS_NAME_SYSTEM' 'Valid system var name'),
3: ('SPSS_NAME_BADLTH', badLengthMsg),
4: ('SPSS_NAME_BADCHAR', 'Invalid character or embedded blank'),
5: ('SPSS_NAME_RESERVED', 'Name is a reserved word'),
6: ('SPSS_NAME_BADFIRST', 'Invalid initial char (otherwise OK)')}
validate = self.spssio.spssValidateVarname
func = self.spssio.spssSetVarName
for varName in self.varNames:
varLength = self.varTypes[varName]
retcode = validate(c_char_p(varName))
if retcode > 0:
msg = "%r is an invalid variable name [%r]" % \
(varName, ": ".join(varNameRetcodes.get(retcode)))
raise SPSSIOError(msg, retcode)
retcode = func(c_int(self.fh), c_char_p(varName), c_int(varLength))
if retcode > 0:
msg = "Problem setting variable name %r" % varName
raise SPSSIOError(msg, retcode)
@property
@decode
def valueLabels(self):
"""Get/Set VALUE LABELS.
Takes a dictionary of the form {varName: {value: valueLabel}:
--{'numGender': {1: 'female', {2: 'male'}}
--{'strGender': {'f': 'female', 'm': 'male'}}"""
def initArrays(isNumeric=True, size=1000):
"""default size assumes never more than 1000 labels"""
labelsArr = (POINTER(c_char_p * size))()
if isNumeric:
return (POINTER(c_double * size))(), labelsArr
return (POINTER(c_char_p * size))(), labelsArr
valueLabels = {}
for varName in self.varNames:
vName = self.vNames[varName]
numLabels = c_int()
# step 1: get array size (numeric values)
if self.varTypes[varName] == 0:
valuesArr, labelsArr = initArrays(True)
func = self.spssio.spssGetVarNValueLabels
retcode = func(c_int(self.fh), c_char_p(vName),
byref(valuesArr), byref(labelsArr),
byref(numLabels))
valuesArr, labelsArr = initArrays(True, numLabels.value)
# step 1: get array size (string values)
else:
valuesArr, labelsArr = initArrays(False)
func = self.spssio.spssGetVarCValueLabels
retcode = func(c_int(self.fh), c_char_p(vName),
byref(valuesArr), byref(labelsArr),
byref(numLabels))
valuesArr, labelsArr = initArrays(False, numLabels.value)
# step 2: get labels with array of proper size
retcode = func(c_int(self.fh), c_char_p(vName), byref(valuesArr),
byref(labelsArr), byref(numLabels))
if retcode > 0:
msg = "Error getting value labels of variable %r"
raise SPSSIOError(msg % varName, retcode)
# get array contents
if not numLabels.value:
continue
values = [valuesArr[0][i] for i in xrange(numLabels.value)]
labels = [labelsArr[0][i] for i in xrange(numLabels.value)]
valueLabelsX = [(val, lbl) for val, lbl in zip(values, labels)]
valueLabels[varName] = dict(valueLabelsX)
# clean up
args = (valuesArr, labelsArr, numLabels)
if self.varTypes[varName] == 0:
self.freeMemory("spssFreeVarNValueLabels", *args)
else:
self.freeMemory("spssFreeVarCValueLabels", *args)
return valueLabels
@valueLabels.setter
def valueLabels(self, valueLabels):
if not valueLabels:
return
valLabN = self.spssio.spssSetVarNValueLabel
valLabC = self.spssio.spssSetVarCValueLabel
valueLabels = self.encode(valueLabels)
for varName, valueLabelsX in valueLabels.iteritems():
valueLabelsX = self.encode(valueLabelsX)
for value, label in valueLabelsX.iteritems():
if self.varTypes[varName] == 0:
retcode = valLabN(c_int(self.fh), c_char_p(varName),
c_double(value), c_char_p(label))
else:
retcode = valLabC(c_int(self.fh), c_char_p(varName),
c_char_p(value), c_char_p(label))
if retcode > 0:
msg = "Problem with setting value labels of variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def varLabels(self):
"""Get/set VARIABLE LABELS.
Returns/takes a dictionary of the form {varName: varLabel}.
For example: varLabels = {'salary': 'Salary (dollars)',
'educ': 'Educational level (years)'}"""
SPSS_VARLABEL_SHORT = 120 # fixed amount
funcS = self.spssio.spssGetVarLabel
funcL = self.spssio.spssGetVarLabelLong
varLabels = {}
for varName in self.varNames:
varLabel = create_string_buffer(SPSS_VARLABEL_SHORT + 1)
vName = self.vNames[varName]
retcode = funcS(c_int(self.fh), c_char_p(vName), byref(varLabel))
if varLabel.value and len(varLabel.value) > SPSS_VARLABEL_SHORT:
lenBuff = MAXLENGTHS['SPSS_MAX_VARLABEL']
varLabel = create_string_buffer(lenBuff)
retcode = funcL(c_int(self.fh), c_char_p(varName),
byref(varLabel), c_int(lenBuff), byref(c_int))
varLabels[varName] = varLabel.value
if retcode > 0:
msg = "Error getting variable label of variable %r" % varName
raise SPSSIOError(msg, retcode)
return varLabels
@varLabels.setter
def varLabels(self, varLabels):
if not varLabels:
return
func = self.spssio.spssSetVarLabel
varLabels = self.encode(varLabels)
for varName, varLabel in varLabels.iteritems():
retcode = func(c_int(self.fh), c_char_p(varName),
c_char_p(varLabel))
if retcode > 0:
msg = ("Problem with setting variable label %r of variable %r"
% (varLabel, varName))
raise SPSSIOError(msg, retcode)
@property
@decode
def formats(self):
"""Get the PRINT FORMATS, set PRINT and WRITE FORMATS.
Returns/takes a dictionary of the form {varName: <format_>.
For example: formats = {'salary': 'DOLLAR8', 'gender': 'A1',
'educ': 'F8.2'}"""
if hasattr(self, "varformats"):
return self.varformats
printFormat_, printDec_, printWid_ = c_int(), c_int(), c_int()
func = self.spssio.spssGetVarPrintFormat
formats = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(printFormat_), byref(printDec_),
byref(printWid_))
if retcode > 0:
msg = "Error getting print format for variable %r"
raise SPSSIOError(msg % vName, retcode)
printFormat = allFormats.get(printFormat_.value)[0]
printFormat = printFormat.split("_")[-1]
format_ = printFormat + str(printWid_.value)
if self.varTypes[varName] == 0:
format_ += ("." + str(printDec_.value))
if format_.endswith(".0"):
format_ = format_[:-2]
formats[varName] = format_
return formats
def _splitformats(self):
"""This function returns the 'bare' formats + variable widths,
e.g. format_ F5.3 is returned as 'F' and '5'"""
regex = re.compile("\w+(?P<varWid>\d+)[.]?\d?", re.I)
bareformats, varWids = {}, {}
for varName, format_ in self.formats.iteritems():
bareformats[varName] = re.sub(r"\d+.", "", format_)
varWids[varName] = int(regex.search(format_).group("varWid"))
return bareformats, varWids
@formats.setter
def formats(self, formats):
if not formats:
return
reverseFormats = dict([(v[0][9:], k) for k, v in allFormats.items()])
validValues = sorted(reverseFormats.keys())
regex = "(?P<printFormat>A(HEX)?)(?P<printWid>\d+)"
isStringVar = re.compile(regex, re.IGNORECASE)
regex = "(?P<printFormat>[A-Z]+)(?P<printWid>\d+)\.?(?P<printDec>\d*)"
isAnyVar = re.compile(regex, re.IGNORECASE)
funcP = self.spssio.spssSetVarPrintFormat # print type
funcW = self.spssio.spssSetVarWriteFormat # write type
for varName, format_ in self.encode(formats).iteritems():
format_ = format_.upper()
gotString = isStringVar.match(format_)
gotAny = isAnyVar.match(format_)
msg = ("Unknown format_ %r for variable %r. " +
"Valid formats are: %s")
msg = msg % (", ".join(validValues), format_, varName)
if gotString:
printFormat = gotString.group("printFormat")
printFormat = reverseFormats.get(printFormat)
printDec = 0
printWid = int(gotString.group("printWid"))
elif gotAny:
printFormat = gotAny.group("printFormat")
printFormat = reverseFormats.get(printFormat)
printDec = gotAny.group("printDec")
printDec = int(printDec) if printDec else 0
printWid = int(gotAny.group("printWid"))
else:
raise ValueError(msg)
if printFormat is None:
raise ValueError(msg)
args = (c_int(self.fh), c_char_p(varName), c_int(printFormat),
c_int(printDec), c_int(printWid))
retcode1, retcode2 = funcP(*args), funcW(*args)
if retcodes.get(retcode1) == "SPSS_INVALID_PRFOR":
# invalid PRint FORmat
msg = "format_ for %r misspecified (%r)"
raise SPSSIOError(msg % (varName, format_), retcode1)
elif retcode1 > 0:
msg = "Error setting format_ %r for %r"
raise SPSSIOError(msg % (format_, varName), retcode1)
def _getMissingValue(self, varName):
"""This is a helper function for the missingValues getter
method. The function returns the missing values of variable <varName>
as a a dictionary. The dictionary keys and items depend on the
particular definition, which may be discrete values and/or ranges.
Range definitions are only possible for numerical variables."""
if self.varTypes[varName] == 0:
func = self.spssio.spssGetVarNMissingValues
args = (c_double(), c_double(), c_double())
else:
func = self.spssio.spssGetVarCMissingValues
args = (create_string_buffer(9), create_string_buffer(9),
create_string_buffer(9))
missingFmt = c_int()
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(missingFmt), *map(byref, args))
if retcode > 0:
msg = "Error getting missing value for variable %r"
raise SPSSIOError(msg % varName, retcode)
v1, v2, v3 = [v.value for v in args]
userMiss = dict([(v, k) for k, v in userMissingValues.iteritems()])
missingFmt = userMiss[missingFmt.value]
if missingFmt == "SPSS_NO_MISSVAL":
return {}
elif missingFmt == "SPSS_ONE_MISSVAL":
return {"values": [v1]}
elif missingFmt == "SPSS_TWO_MISSVAL":
return {"values": [v1, v2]}
elif missingFmt == "SPSS_THREE_MISSVAL":
return {"values": [v1, v2, v3]}
elif missingFmt == "SPSS_MISS_RANGE":
return {"lower": v1, "upper": v2}
elif missingFmt == "SPSS_MISS_RANGEANDVAL":
return {"lower": v1, "upper": v2, "value": v3}
def _setMissingValue(self, varName, **kwargs):
"""This is a helper function for the missingValues setter
method. The function sets missing values for variable <varName>.
Valid keyword arguments are:
* to specify a RANGE: 'lower', 'upper', optionally with 'value'
* to specify DISCRETE VALUES: 'values', specified as a list no longer
than three items, or as None, or as a float/int/str
"""
if kwargs == {}:
return 0
fargs = ["lower", "upper", "value", "values"]
if set(kwargs.keys()).difference(set(fargs)):
raise ValueError("Allowed keywords are: %s" % ", ".join(fargs))
varName = self.encode(varName)
varType = self.varTypes[varName]
# range of missing values, e.g. MISSING VALUES aNumVar (-9 THRU -1).
if varType == 0:
placeholder = 0.0
if "lower" in kwargs and "upper" in kwargs and "value" in kwargs:
missingFmt = "SPSS_MISS_RANGEANDVAL"
args = kwargs["lower"], kwargs["upper"], kwargs["value"]
elif "lower" in kwargs and "upper" in kwargs:
missingFmt = "SPSS_MISS_RANGE"
args = kwargs["lower"], kwargs["upper"], placeholder
else:
placeholder, args = "0", None
# up to three discrete missing values
if "values" in kwargs:
values = self.encode(kwargs.values()[0])
if isinstance(values, (float, int, str)):
values = [values]
# check if missing values strings values are not too long
strMissLabels = [len(v) for v in values if isinstance(v, str)]
if strMissLabels and max(strMissLabels) > 9:
raise ValueError("Missing value label > 9 bytes")
nvalues = len(values) if values is not None else values
if values is None or values == {}:
missingFmt = "SPSS_NO_MISSVAL"
args = placeholder, placeholder, placeholder
elif nvalues == 1:
missingFmt = "SPSS_ONE_MISSVAL"
args = values + [placeholder, placeholder]
elif nvalues == 2:
missingFmt = "SPSS_TWO_MISSVAL"
args = values + [placeholder]
elif nvalues == 3:
missingFmt = "SPSS_THREE_MISSVAL"
args = values
else:
msg = "You can specify up to three individual missing values"
raise ValueError(msg)
# numerical vars
if varType == 0 and args:
func = self.spssio.spssSetVarNMissingValues
func.argtypes = [c_int, c_char_p, c_int,
c_double, c_double, c_double]
args = map(float, args)
# string vars
else:
if args is None:
raise ValueError("Illegal keyword for character variable")
func = self.spssio.spssSetVarCMissingValues
func.argtypes = [c_int, c_char_p, c_int,
c_char_p, c_char_p, c_char_p]
retcode = func(self.fh, varName, userMissingValues[missingFmt], *args)
if retcode > 0:
msg = "Problem setting missing value of variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def missingValues(self):
"""Get/Set MISSING VALUES.
User missing values are values that will not be included in
calculations by SPSS. For example, 'don't know' might be coded as a
user missing value (a value of 999 is typically used, so when vairable
'age' has values 5, 15, 999, the average age is 10). This is
different from 'system missing values', which are blank/null values.
Takes a dictionary of the following form:
{'someNumvar1': {'values': [999, -1, -2]}, # discrete values
'someNumvar2': {'lower': -9, 'upper':-1}, # range "-9 THRU -1"
'someNumvar3': {'lower': -9, 'upper':-1, 'value': 999},
'someStrvar1': {'values': ['foo', 'bar', 'baz']},
'someStrvar2': {'values': 'bletch'}} # shorthand """
missingValues = {}
for varName in self.varNames:
missingValues[varName] = self._getMissingValue(varName)
return missingValues
@missingValues.setter
def missingValues(self, missingValues):
if missingValues:
for varName, kwargs in missingValues.iteritems():
self._setMissingValue(varName, **kwargs)
# measurelevel, colwidth and alignment must all be set or not at all.
@property
@decode
def measureLevels(self):
"""Get/Set VARIABLE LEVEL (measurement level).
Returns/Takes a dictionary of the form {varName: varMeasureLevel}.
Valid measurement levels are: "unknown", "nominal", "ordinal", "scale",
"ratio", "flag", "typeless". This is used in Spss procedures such as
CTABLES."""
func = self.spssio.spssGetVarMeasureLevel
levels = {0: "unknown", 1: "nominal", 2: "ordinal", 3: "scale",
3: "ratio", 4: "flag", 5: "typeless"}
measureLevel = c_int()
varMeasureLevels = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(measureLevel))
varMeasureLevels[varName] = levels.get(measureLevel.value)
if retcode > 0:
msg = "Error getting variable measurement level: %r"
raise SPSSIOError(msg % varName, retcode)
return varMeasureLevels
@measureLevels.setter
def measureLevels(self, varMeasureLevels):
if not varMeasureLevels:
return
func = self.spssio.spssSetVarMeasureLevel
levels = {"unknown": 0, "nominal": 1, "ordinal": 2, "scale": 3,
"ratio": 3, "flag": 4, "typeless": 5}
for varName, level in self.encode(varMeasureLevels).iteritems():
if level.lower() not in levels:
msg = "Valid levels are %"
raise ValueError(msg % ", ".join(levels.keys()))
level = levels.get(level.lower())
retcode = func(c_int(self.fh), c_char_p(varName), c_int(level))
if retcode > 0:
msg = ("Error setting variable mesasurement level. " +
"Valid levels are: %s")
raise SPSSIOError(msg % ", ".join(levels.keys()), retcode)
@property
@decode
def columnWidths(self):
"""Get/Set VARIABLE WIDTH (display width).
Returns/Takes a dictionary of the form {varName: <int>}. A value of
zero is special and means that the IBM SPSS Statistics Data Editor
is to set an appropriate width using its own algorithm. If used,
variable alignment, measurement level and column width all needs to
be set."""
func = self.spssio.spssGetVarColumnWidth
varColumnWidth = c_int()
varColumnWidths = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(varColumnWidth))
if retcode > 0:
msg = ("Error getting column width: %r" % varName)
raise SPSSIOError(msg, retcode)
varColumnWidths[varName] = varColumnWidth.value
return varColumnWidths
@columnWidths.setter
def columnWidths(self, varColumnWidths):
if not varColumnWidths:
return
func = self.spssio.spssSetVarColumnWidth
for varName, varColumnWidth in varColumnWidths.iteritems():
retcode = func(c_int(self.fh), c_char_p(varName),
c_int(varColumnWidth))
if retcode > 0:
msg = "Error setting variable colunm width"
raise SPSSIOError(msg, retcode)
def _setColWidth10(self):
"""Set the variable display width of string values to at least 10
(it's annoying that SPSS displays e.g. a one-character variable in
very narrow columns). This also sets all measurement levels to
"unknown" and all variable alignments to "left". This function is
only called if column widths, measurement levels and variable
alignments are None."""
columnWidths = {}
for varName, varType in self.varTypes.iteritems():
# zero = appropriate width determined by spss
columnWidths[varName] = 10 if 0 < varType < 10 else 0
self.columnWidths = columnWidths
self.measureLevels = dict([(v, "unknown") for v in self.varNames])
self.alignments = dict([(v, "left") for v in self.varNames])
@property
@decode
def alignments(self):
"""Get/Set VARIABLE ALIGNMENT.
Returns/Takes a dictionary of the form {varName: alignment}
Valid alignment values are: left, right, center. If used, variable
alignment, measurement level and column width all need to be set.
"""
func = self.spssio.spssGetVarAlignment
alignments = {0: "left", 1: "right", 2: "center"}
alignment_ = c_int()
varAlignments = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(alignment_))
alignment = alignments[alignment_.value]
varAlignments[varName] = alignment
if retcode > 0:
msg = ("Error getting variable alignment: %r" % varName)
raise SPSSIOError(msg, retcode)
return varAlignments
@alignments.setter
def alignments(self, varAlignments):
if not varAlignments:
return
func = self.spssio.spssSetVarAlignment
alignments = {"left": 0, "right": 1, "center": 2}
for varName, varAlignment in varAlignments.iteritems():
if varAlignment.lower() not in alignments:
msg = "Valid alignments are %"
raise ValueError(msg % ", ".join(alignments.keys()))
alignment = alignments.get(varAlignment.lower())
retcode = func(c_int(self.fh), c_char_p(varName), c_int(alignment))
if retcode > 0:
msg = "Error setting variable alignment for variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def varSets(self):
"""Get/Set VARIABLE SET information.
Returns/Takes a dictionary with SETNAME as keys and a list of SPSS
variables as values. For example: {'SALARY': ['salbegin',
'salary'], 'DEMOGR': ['gender', 'minority', 'educ']}"""
varSets = c_char_p()
func = self.spssio.spssGetVariableSets
retcode = func(c_int(self.fh), byref(varSets))
if retcode > 0:
msg = "Problem getting variable set information"
raise SPSSIOError(msg, retcode)
if not varSets.value:
return {}
varSets_ = {}
for varSet in varSets.value.split("\n")[:-1]:
k, v = varSet.split("= ")
varSets_[k] = v.split()
# clean up
self.freeMemory("spssFreeVariableSets", varSets)
return varSets_
@varSets.setter
def varSets(self, varSets):
if not varSets:
return
varSets_ = []
for varName, varSet in varSets.iteritems():
varSets_.append("%s= %s" % (varName, " ".join(varSet)))
varSets_ = c_char_p("\n".join(varSets_))
retcode = self.spssio.spssSetVariableSets(c_int(self.fh), varSets_)
if retcode > 0:
msg = "Problem setting variable set information"
raise SPSSIOError(msg, retcode)
@property
@decode
def varRoles(self):
"""Get/Set VARIABLE ROLES.
Returns/Takes a dictionary of the form {varName: varRole}, where
varRoles may be any of the following: 'both', 'frequency', 'input',
'none', 'partition', 'record ID', 'split', 'target'"""
func = self.spssio.spssGetVarRole
roles = {0: "input", 1: "target", 2: "both", 3: "none", 4: "partition",
5: "split", 6: "frequency", 7: "record ID"}
varRoles = {}
varRole_ = c_int()
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName), byref(varRole_))
varRole = roles.get(varRole_.value)
varRoles[varName] = varRole
if retcode > 0:
msg = "Problem getting variable role for variable %r"
raise SPSSIOError(msg % varName, retcode)
return varRoles
@varRoles.setter
def varRoles(self, varRoles):
if not varRoles:
return
roles = {"input": 0, "target": 1, "both": 2, "none": 3, "partition": 4,
"split": 5, "frequency": 6, "record ID": 7}
func = self.spssio.spssSetVarRole
for varName, varRole in varRoles.iteritems():
varRole = roles.get(varRole)
retcode = func(c_int(self.fh), c_char_p(varName), c_int(varRole))
if retcode > 0:
msg = "Problem setting variable role %r for variable %r"
raise SPSSIOError(msg % (varRole, varName), retcode)
@property
@decode
def varAttributes(self):
"""Get/Set VARIABLE ATTRIBUTES.
Returns/Takes dictionary of the form:
{'var1': {'attr name x': 'attr value x','attr name y': 'attr value y'},
'var2': {'attr name a': 'attr value a','attr name b': 'attr value b'}}
"""
# abbreviation for readability and speed
func = self.spssio.spssGetVarAttributes
# initialize arrays
MAX_ARRAY_SIZE = 1000
attrNamesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
attrValuesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
attributes = {}
for varName in self.varNames:
vName = self.vNames[varName]
# step 1: get array size
nAttr = c_int()
retcode = func(c_int(self.fh), c_char_p(vName),
byref(attrNamesArr), byref(attrValuesArr),
byref(nAttr))
if retcode > 0:
msg = "@Problem getting attributes of variable %r (step 1)"
raise SPSSIOError(msg % varName, retcode)
# step 2: get attributes with arrays of proper size
nAttr = c_int(nAttr.value)
attrNamesArr = (POINTER(c_char_p * nAttr.value))()
attrValuesArr = (POINTER(c_char_p * nAttr.value))()
retcode = func(c_int(self.fh), c_char_p(vName),
byref(attrNamesArr), byref(attrValuesArr),
byref(nAttr))
if retcode > 0:
msg = "Problem getting attributes of variable %r (step 2)"
raise SPSSIOError(msg % varName, retcode)
# get array contents
if not nAttr.value:
continue
k, v, n = attrNamesArr[0], attrValuesArr[0], nAttr.value
attribute = dict([(k[i], v[i]) for i in xrange(n)])
attributes[varName] = attribute
# clean up
args = (attrNamesArr, attrValuesArr, nAttr)
self.freeMemory("spssFreeAttributes", *args)
return attributes
@varAttributes.setter
def varAttributes(self, varAttributes):
if not varAttributes:
return
func = self.spssio.spssSetVarAttributes
for varName in self.varNames:
attributes = varAttributes.get(varName)
if not attributes:
continue
nAttr = len(attributes)
attrNames = (c_char_p * nAttr)(*attributes.keys())
attrValues = (c_char_p * nAttr)(*attributes.values())
retcode = func(c_int(self.fh), c_char_p(varName),
pointer(attrNames), pointer(attrValues),
c_int(nAttr))
if retcode > 0:
msg = "Problem setting variable attributes for variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def fileAttributes(self):
"""Get/Set DATAFILE ATTRIBUTES.
Returns/Takes a dictionary of the form:
{'attrName[1]': 'attrValue1', 'revision[1]': '2010-10-09',
'revision[2]': '2010-10-22', 'revision[3]': '2010-11-19'}
"""
# abbreviation for readability
func = self.spssio.spssGetFileAttributes
# step 1: get array size
MAX_ARRAY_SIZE = 100 # assume never more than 100 file attributes
attrNamesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
attrValuesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
nAttr = c_int()
retcode = func(c_int(self.fh), byref(attrNamesArr),
byref(attrValuesArr), byref(nAttr))
# step 2: get attributes with arrays of proper size
nAttr = c_int(nAttr.value)
attrNamesArr = (POINTER(c_char_p * nAttr.value))()
attrValuesArr = (POINTER(c_char_p * nAttr.value))()
retcode = func(c_int(self.fh), byref(attrNamesArr),
byref(attrValuesArr), byref(nAttr))
if retcode > 0:
raise SPSSIOError("Problem getting file attributes", retcode)
# get array contents
if not nAttr.value:
return {}
k, v = attrNamesArr[0], attrValuesArr[0]
attributes = dict([(k[i], v[i]) for i in xrange(nAttr.value)])
# clean up
args = (attrNamesArr, attrValuesArr, nAttr)
self.freeMemory("spssFreeAttributes", *args)
return attributes
@fileAttributes.setter
def fileAttributes(self, fileAttributes):
if not fileAttributes:
return
attributes, valueLens = {}, []
for name, values in fileAttributes.iteritems():
valueLens.append(len(values))
for value in values:
attributes[name] = value
#nAttr = len(fileAttributes)
nAttr = max(valueLens) # n elements per vector. But this may vary??
attrNames = (c_char_p * nAttr)(*attributes.keys())
attrValues = (c_char_p * nAttr)(*attributes.values())
func = self.spssio.spssSetFileAttributes
retcode = func(c_int(self.fh), pointer(attrNames),
pointer(attrValues), c_int(nAttr))
if retcode > 0:
raise SPSSIOError("Problem setting file attributes", retcode)
def _getMultRespDef(self, mrDef):
"""Get 'normal' multiple response defintions.
This is a helper function for the multRespDefs getter function.
A multiple response definition <mrDef> in the string format returned
by the IO module is converted into a multiple response definition of
the form multRespSet = {<setName>: {"setType": <setType>, "label":
<lbl>, "varNames": <list_of_varNames>}}. SetType may be either 'D'
(multiple dichotomy sets) or 'C' (multiple category sets). If setType
is 'D', the multiple response definition also includes '"countedValue":
countedValue'"""
regex = "\$(?P<setName>\w+)=(?P<setType>[CD])\n?"
m = re.search(regex + ".*", mrDef, re.I | re.U)
if not m:
return {}
setType = m.group("setType")
if setType == "C": # multiple category sets
regex += " (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?"
matches = re.findall(regex, mrDef, re.I)
setName, setType, lblLen, lblVarNames = matches[0]
else: # multiple dichotomy sets
regex += ("(?P<valueLen>\d+) (?P<countedValue>\w+)" +
" (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?")
matches = re.findall(regex, mrDef, re.I | re.U)
setName, setType, valueLen = matches[0][:3]
countedValue, lblLen, lblVarNames = matches[0][3:]
lbl = lblVarNames[:int(lblLen)]
varNames = lblVarNames[int(lblLen):].split()
multRespSet = {setName: {"setType": setType, "label": lbl,
"varNames": varNames}}
if setType == "D":
multRespSet[setName]["countedValue"] = countedValue
return multRespSet
def _setMultRespDefs(self, multRespDefs):
"""Set 'normal' multiple response defintions.
This is a helper function for the multRespDefs setter function.
It translates the multiple response definition, specified as a
dictionary, into a string that the IO module can use"""
mrespDefs = []
for setName, rest in multRespDefs.iteritems():
print setName, rest, rest["setType"]
if rest["setType"] not in ("C", "D"):
continue
rest["setName"] = setName
mrespDef = "$%(setName)s=%(setType)s" % rest
lblLen = len(rest["label"])
rest["lblLen"] = lblLen
rest["varNames"] = " ".join(rest["varNames"])
tail = "%(varNames)s" if lblLen == 0 else "%(label)s %(varNames)s"
if rest["setType"] == "C": # multiple category sets
template = " %%(lblLen)s %s " % tail
else: # multiple dichotomy sets
template = " %%(countedValue)s %%(lblLen)s %s " % tail
mrespDef += template % rest
mrespDefs.append(mrespDef)
mrespDefs = "\n".join(mrespDefs)
return mrespDefs
def _getMultRespDefsEx(self, mrDef):
"""Get 'extended' multiple response defintions.
This is a helper function for the multRespDefs getter function."""
regex = ("\$(?P<setName>\w+)=(?P<setType>E) (?P<flag1>1)" +
"(?P<flag2>1)? (?P<valueLen>[0-9]+) (?P<countedValue>\w+) " +
"(?P<lblLen>[0-9]+) (?P<lblVarNames>[\w ]+)")
matches = re.findall(regex, mrDef, re.I | re.U)
setName, setType, flag1, flag2 = matches[0][:4]
valueLen, countedValue, lblLen, lblVarNames = matches[0][4:]
length = int(lblLen)
label, varNames = lblVarNames[:length], lblVarNames[length:].split()
return {setName: {"setType": setType, "firstVarIsLabel": bool(flag2),
"label": label, "countedValue": countedValue,
"varNames": varNames}}
def _setMultRespDefsEx(self, multRespDefs):
"""Set 'extended' multiple response defintions.
This is a helper function for the multRespDefs setter function."""
mrDefs = []
for setName, rest in multRespDefs.iteritems():
if rest["setType"] != "E":
continue
rest["setName"] = setName
v = int(rest["firstVarIsLabel"])
rest["firstVarIsLabel"] = v if v == 1 else ""
rest["valueLen"] = len(rest["countedValue"])
rest["lblLen"] = len(rest["label"])
rest["varNames"] = " ".join(rest["varNames"])
mrDef = "$%(setName)s=%(setType)s 1%(firstVarIsLabel)s "
mrDef += "%(valueLen)s %(countedValue)s %(lblLen)s %(label)s "
mrDef += "%(varNames)s"
mrDefs.append((mrDef % rest).replace(" ", " "))
return "\n".join(mrDefs)
@property
@decode
def multRespDefs(self):
"""Get/Set MRSETS (multiple response) sets.
Returns/takes a dictionary of the form:
--multiple category sets: {setName: {"setType": "C", "label": lbl,
"varNames": [<list_of_varNames>]}}
--multiple dichotomy sets: {setName: {"setType": "D", "label": lbl,
"varNames": [<list_of_varNames>], "countedValue": countedValue}}
--extended multiple dichotomy sets: {setName: {"setType": "E",
"label": lbl, "varNames": [<list_of_varNames>], "countedValue":
countedValue, 'firstVarIsLabel': <bool>}}
For example:
categorical = {"setType": "C", "label": "labelC",
"varNames": ["salary", "educ"]}
dichotomous1 = {"setType": "D", "label": "labelD",
"varNames": ["salary", "educ"], "countedValue": "Yes"}
dichotomous2 = {"setType": "D", "label": "", "varNames":
["salary", "educ", "jobcat"], "countedValue": "No"}
extended1 = {"setType": "E", "label": "", "varNames": ["mevar1",
"mevar2", "mevar3"], "countedValue": "1",
"firstVarIsLabel": True}
extended2 = {"setType": "E", "label":
"Enhanced set with user specified label", "varNames":
["mevar4", "mevar5", "mevar6"], "countedValue":
"Yes", "firstVarIsLabel": False}
multRespDefs = {"testSetC": categorical, "testSetD1": dichotomous1,
"testSetD2": dichotomous2, "testSetEx1": extended1,
"testSetEx2": extended2}
"""
## Normal Multiple response definitions
func = self.spssio.spssGetMultRespDefs
mrDefs = c_char_p()
retcode = func(c_int(self.fh), pointer(mrDefs))
if retcode > 0:
msg = "Problem getting multiple response definitions"
raise SPSSIOError(msg, retcode)
multRespDefs = {}
if mrDefs.value:
for mrDef in mrDefs.value.split("\n"):
for setName, rest in self._getMultRespDef(mrDef).iteritems():
multRespDefs[setName] = rest
self.freeMemory("spssFreeMultRespDefs", mrDefs)
## Extended Multiple response definitions
mrDefsEx = c_char_p()
func = self.spssio.spssGetMultRespDefsEx
retcode = func(c_int(self.fh), pointer(mrDefsEx))
if retcode > 0:
msg = "Problem getting extended multiple response definitions"
raise SPSSIOError(msg, retcode)
multRespDefsEx = {}
if mrDefsEx.value:
for mrDefEx in mrDefsEx.value.split("\n"):
for setName, rest in self._getMultRespDef(mrDefEx).iteritems():
multRespDefsEx[setName] = rest
self.freeMemory("spssFreeMultRespDefs", mrDefsEx)
multRespDefs.update(multRespDefsEx)
return multRespDefs
@multRespDefs.setter
def multRespDefs(self, multRespDefs):
if not multRespDefs:
return
normal = self._setMultRespDefs(multRespDefs)
extended = self._setMultRespDefsEx(multRespDefs)
if normal and extended:
combinedDefs = normal + " \n" + extended
elif normal and not extended:
combinedDefs = normal
elif extended and not normal:
combinedDefs = extended
func = self.spssio.spssSetMultRespDefs
retcode = func(c_int(self.fh), c_char_p(combinedDefs))
if retcode > 0:
msg = "Problem setting multiple response definitions"
raise SPSSIOError(msg, retcode)
@property
@decode
def caseWeightVar(self):
"""Get/Set WEIGHT variable.
Takes a valid varName, and returns weight variable, if any, as a
string."""
varNameBuff = create_string_buffer(65)
func = self.spssio.spssGetCaseWeightVar
retcode = func(c_int(self.fh), byref(varNameBuff))
if retcode > 0:
msg = "Problem getting case weight variable name"
raise SPSSIOError(msg, retcode)
return varNameBuff.value
@caseWeightVar.setter
def caseWeightVar(self, varName):
if not varName:
return
func = self.spssio.spssSetCaseWeightVar
retcode = func(c_int(self.fh), c_char_p(varName))
if retcode > 0:
msg = "Problem setting case weight variable name %r" % varName
raise SPSSIOError(msg, retcode)
@property
@decode
def dateVariables(self): # seems to be okay
"""Get/Set DATE information. This function reports the Forecasting
(Trends) date variable information, if any, in IBM SPSS Statistics
data files. Entirely untested and not implemented in reader/writer"""
# step 1: get array size
nElements = c_int()
func = self.spssio.spssGetDateVariables
MAX_ARRAY_SIZE = 100
dateInfoArr = (POINTER(c_long * MAX_ARRAY_SIZE))()
retcode = func(c_int(self.fh), byref(nElements), byref(dateInfoArr))
# step 2: get date info with array of proper size
dateInfoArr = (POINTER(c_long * nElements.value))()
retcode = func(c_int(self.fh), byref(nElements), byref(dateInfoArr))
if retcode > 0:
raise SPSSIOError("Error getting TRENDS information", retcode)
# get array contents
nElem = nElements.value
if not nElem:
return {}
dateInfo = [dateInfoArr[0][i] for i in xrange(nElem)]
fixedDateInfo = dateInfo[:6]
otherDateInfo = [dateInfo[i: i + 3] for i in xrange(6, nElem, 3)]
dateInfo = {"fixedDateInfo": fixedDateInfo,
"otherDateInfo": otherDateInfo}
# clean up
self.freeMemory("spssFreeDateVariables", dateInfoArr)
return dateInfo
@dateVariables.setter
def dateVariables(self, dateInfo): # entirely untested!
dateInfo = dateInfo["fixedDateInfo"] + dateInfo["otherDateInfo"]
dateInfo = reduce(list.__add__, dateInfo) # flatten list
isAllFloats = all([isinstance(d, float) for d in dateInfo])
isSixPlusTriplets = (len(dateInfo) - 6) % 3 == 0
if not isAllFloats and isSixPlusTriplets:
msg = ("TRENDS date info must consist of 6 fixed elements" +
"+ <nCases> three-element groups of other date info " +
"(all floats)")
raise TypeError(msg)
func = self.spssio.spssSetDateVariables
dateInfoArr = (nElements * c_long)(*dateInfo)
retcode = func(c_int(self.fh), c_int(nElements), dateInfoArr)
if retcode > 0:
raise SPSSIOError("Error setting TRENDS information", retcode)
@property
@decode
def textInfo(self):
"""Get/Set text information.
Takes a savFileName and returns a string of the form: "File %r built
using SavReaderWriter.py version %s (%s)". This is akin to, but
*not* equivalent to the SPSS syntax command DISPLAY DOCUMENTS"""
textInfo = create_string_buffer(256)
retcode = self.spssio.spssGetTextInfo(c_int(self.fh), byref(textInfo))
if retcode > 0:
raise SPSSIOError("Error getting textInfo", retcode)
return textInfo.value
@textInfo.setter
def textInfo(self, savFileName):
info = (os.path.basename(savFileName), __version__, time.asctime())
textInfo = "File '%s' built using SavReaderWriter.py version %s (%s)"
textInfo = textInfo % info
if self.ioUtf8 and isinstance(savFileName, unicode):
textInfo = textInfo.encode("utf-8")
func = self.spssio.spssSetTextInfo
retcode = func(c_int(self.fh), c_char_p(textInfo[:256]))
if retcode > 0:
raise SPSSIOError("Error setting textInfo", retcode)
@property
@decode
def fileLabel(self):
"""Get/Set FILE LABEL (id string)
Takes a file label (basestring), and returns file label, if any, as
a string."""
idStr = create_string_buffer(65)
retcode = self.spssio.spssGetIdString(c_int(self.fh), byref(idStr))
if retcode > 0:
raise SPSSIOError("Error getting file label (id string)", retcode)
return idStr.value
@fileLabel.setter
def fileLabel(self, idStr):
if idStr is None:
idStr = "File created by user %r at %s"[:64] % \
(getpass.getuser(), time.asctime())
if self.ioUtf8 and isinstance(idStr, unicode):
idStr = idStr.encode("utf-8")
retcode = self.spssio.spssSetIdString(c_int(self.fh), c_char_p(idStr))
if retcode > 0:
raise SPSSIOError("Error setting file label (id string)", retcode)
class SavHeaderReader(Header):
"""
This class contains methods that read the data dictionary of an SPSS
data file. This yields the same information as the Spss command 'DISPLAY
DICTIONARY' NB: do not confuse an Spss dictionary with a Python
dictionary!
Typical use:
with SavHeaderReader(savFileName) as spssDict:
wholeDict = spssDict.dataDictionary()
print unicode(spssDict)
"""
def __init__(self, savFileName, ioUtf8=False, ioLocale=None):
""" Constructor. Initializes all vars that can be recycled """
super(SavHeaderReader, self).__init__(savFileName, "rb", None,
ioUtf8, ioLocale)
self.fh = self.openSavFile()
self.varNames, self.varTypes = self.varNamesTypes
self.numVars = self.numberofVariables
self.nCases = self.numberofCases
def __str__(self):
""" This function returns a report of the SPSS data dictionary
(i.e., the header), in the encoding of the spss file"""
return unicode(self).encode(self.fileEncoding)
def __unicode__(self):
""" This function returns a report of the SPSS data dictionary
(i.e., the header)."""
report = ""
if self.textInfo:
report += self.textInfo + os.linesep
report += self.reportSpssDataDictionary(self.dataDictionary())
return report
def __enter__(self):
""" This function returns the DictionaryReader object itself so
its methods become available for use with context managers
('with' statements)."""
return self
def __exit__(self, type, value, tb):
""" This function closes the spss data file and does some cleaning."""
if type is not None:
pass # Exception occurred
self.close()
def close(self):
"""This function closes the spss data file and does some cleaning."""
if not segfaults:
self.closeSavFile(self.fh, mode="rb")
def dataDictionary(self):
""" This function returns all the dictionary items. It returns
a Python dictionary based on the Spss dictionary of the given
Spss file. This is equivalent to the Spss command 'DISPLAY
DICTIONARY'."""
items = ["varNames", "varTypes", "valueLabels", "varLabels",
"formats", "missingValues", "measureLevels",
"columnWidths", "alignments", "varSets", "varRoles",
"varAttributes", "fileAttributes", "fileLabel",
"multRespDefs", "caseWeightVar", "multRespDefs"]
if self.ioUtf8:
items = map(unicode, items)
dataDictionary = dict([(item, getattr(self, item)) for item in items])
return dataDictionary
def reportSpssDataDictionary(self, dataDict):
""" This function reports information from the Spss dictionary
of the active Spss dataset. The parameter 'dataDict' is the return
value of dataDictionary()"""
report = []
#import pprint
#pprint.pprint(dataDict)
for kwd, allValues in sorted(dataDict.items()):
report.append("#" + kwd.upper())
if hasattr(allValues, "items"):
for varName, values in allValues.iteritems():
if hasattr(values, "items"):
isList = kwd in ("missingValues", "multRespDefs")
for k, v in sorted(values.iteritems()):
if isList and isinstance(v, (list, tuple)):
vStr = map(unicode.lower, map(unicode, v))
report.append("%s: %s -- %s" %
(varName, k, ", ".join(vStr)))
else:
report.append("%s: %s -- %s" %
(varName, unicode(k).strip(), v))
else:
if isinstance(values, list):
entry = "%s -- %s" % (varName, ", ".join(values))
report.append(entry)
elif values != "":
report.append("%s -- %s" % (varName, values))
else:
if isinstance(allValues, basestring) and allValues:
allValues = [allValues]
for varName in allValues:
report.append(varName)
return os.linesep.join(report)
class SavReader(Header):
""" Read Spss system files (.sav, .zsav)
Parameters:
-savFileName: the file name of the spss data file
-returnHeader: Boolean that indicates whether the first record should
be a list of variable names (default = False)
-recodeSysmisTo: indicates to which value missing values should
be recoded (default = None),
-selectVars: indicates which variables in the file should be selected.
The variables should be specified as a list or a tuple of
valid variable names. If None is specified, all variables
in the file are used (default = None)
-idVar: indicates which variable in the file should be used for use as id
variable for the 'get' method (default = None)
-verbose: Boolean that indicates whether information about the spss data
file (e.g., number of cases, variable names, file size) should be
printed on the screen (default = False).
-rawMode: Boolean that indicates whether values should get SPSS-style
formatting, and whether date variables (if present) should be converted
to ISO-dates. If True, the program does not format any values, which
increases processing speed. (default = False)
-ioUtf8: indicates the mode in which text communicated to or from the
I/O Module will be. Valid values are True (UTF-8 mode aka Unicode mode)
and False (Codepage mode). Cf. SET UNICODE=ON/OFF (default = False)
-ioLocale: indicates the locale of the I/O module. Cf. SET LOCALE (default
= None, which corresponds to locale.getlocale()[0])
Typical use:
savFileName = "d:/someFile.sav"
with SavReader(savFileName, returnHeader=True) as sav:
header = sav.next()
for line in sav:
process(line)
"""
def __init__(self, savFileName, returnHeader=False, recodeSysmisTo=None,
verbose=False, selectVars=None, idVar=None, rawMode=False,
ioUtf8=False, ioLocale=None):
""" Constructor. Initializes all vars that can be recycled """
super(SavReader, self).__init__(savFileName, "rb", None,
ioUtf8, ioLocale)
self.savFileName = savFileName
self.returnHeader = returnHeader
self.recodeSysmisTo = recodeSysmisTo
self.verbose = verbose
self.selectVars = selectVars
self.idVar = idVar
self.rawMode = rawMode
self.header = self.getHeader(self.selectVars)
self.bareformats, self.varWids = self._splitformats()
self.autoRawMode = self._isAutoRawMode()
self.ioUtf8_ = ioUtf8
self.sysmis_ = self.sysmis
self.numVars = self.numberofVariables
self.nCases = self.numberofCases
self.myStruct = self.getStruct(self.varTypes, self.varNames)
self.unpack_from = self.myStruct.unpack_from
self.seekNextCase = self.spssio.spssSeekNextCase
self.caseBuffer = self.getCaseBuffer()
if psycoOk:
self._items = psyco.proxy(self._items) # 3 x faster!
def __enter__(self):
""" This function opens the spss data file (context manager)."""
if self.verbose and self.ioUtf8_:
print unicode(self).replace(os.linesep, "\n")
elif self.verbose:
print str(self).replace(os.linesep, "\n")
return iter(self)
def __exit__(self, type, value, tb):
""" This function closes the spss data file and does some cleaning."""
if type is not None:
pass # Exception occurred
self.close()
def close(self):
"""This function closes the spss data file and does some cleaning."""
if not segfaults:
self.closeSavFile(self.fh, mode="rb")
del self.spssio
def __len__(self):
""" This function reports the number of cases (rows) in the spss data
file. For example: len(SavReader(savFileName))"""
return self.nCases
def __cmp__(self, other):
""" This function implements behavior for all of the comparison
operators so comparisons can be made between SavReader instances,
or comparisons between SavReader instances and integers."""
if not isinstance(other, (SavReader, int)):
raise TypeError
other = other if isinstance(other, int) else len(other)
if len(self) < other:
return -1
elif len(self) == other:
return 0
else:
return 1
def __hash__(self):
"""This function returns a hash value for the object to ensure it
is hashable."""
return id(self)
def __str__(self):
"""This function returns a conscise file report of the spss data file
For example str(SavReader(savFileName))"""
return unicode(self).encode(self.fileEncoding)
def __unicode__(self):
"""This function returns a conscise file report of the spss data file,
For example unicode(SavReader(savFileName))"""
self.fileReport = self.getFileReport(self.savFileName, self.varNames,
self.varTypes, self.formats,
self.nCases)
return self.fileReport
def _isAutoRawMode(self):
"""Helper function for formatValues function. Determines whether
iterating over each individual value is really needed"""
hasDates = bool(set(self.bareformats.values()) & set(supportedDates))
hasNfmt = "N" in self.bareformats
hasRecodeSysmis = self.recodeSysmisTo is not None
items = [hasDates, hasNfmt, hasRecodeSysmis, self.ioUtf8_]
return False if any(items) else True
def formatValues(self, record):
"""This function formats date fields to ISO dates (yyyy-mm-dd), plus
some other date/time formats. The SPSS N format is formatted to a
character value with leading zeroes. System missing values are recoded
to <recodeSysmisTo>. If rawMode==True, this function does nothing"""
if self.rawMode or self.autoRawMode:
return record # 6-7 times faster!
for i, value in enumerate(record):
varName = self.header[i]
varType = self.varTypes[varName]
bareformat_ = self.bareformats[varName]
varWid = self.varWids[varName]
if varType == 0:
# recode system missing values, if present and desired
if value > self.sysmis_:
pass
else:
record[i] = self.recodeSysmisTo
# format N-type values (=numerical with leading zeroes)
if bareformat_ == "N":
#record[i] = str(value).zfill(varWid)
record[i] = "%%0%dd" % varWid % value #15 x faster (zfill)
# convert SPSS dates to ISO dates
elif bareformat_ in supportedDates:
fmt = supportedDates[bareformat_]
args = (value, fmt, self.recodeSysmisTo)
record[i] = self.spss2strDate(*args)
elif varType > 0:
value = value[:varType]
if self.ioUtf8_:
record[i] = value.decode("utf-8")
else:
record[i] = value
return record
def _items(self, start=0, stop=None, step=1, returnHeader=False):
""" This is a helper function to implement the __getitem__ and
the __iter__ special methods. """
if returnHeader:
yield self.header
if all([start == 0, stop is None, step == 1]): # used as iterator
retcode = 0
else:
retcode = self.seekNextCase(c_int(self.fh), c_long(0)) # reset
if stop is None:
stop = self.nCases
selection = self.selectVars is not None
for case in xrange(start, stop, step):
if start:
# only call this when iterating over part of the records
retcode = self.seekNextCase(c_int(self.fh), c_long(case))
if retcode > 0:
raise SPSSIOError("Error seeking case %d" % case, retcode)
elif stop == self.nCases:
self.printPctProgress(case, self.nCases)
record = self.record
if selection:
record = self.selector(record)
record = [record] if len(record) == 1 else list(record)
record = self.formatValues(record)
yield record
def __iter__(self):
"""This function allows the object to be used as an iterator"""
return self._items(0, None, 1, self.returnHeader)
def __getitem__(self, key):
""" This function reports the record of case number <key>.
For example: firstRecord = SavReader(savFileName)[0].
The <key> argument may also be a slice, for example:
firstfiveRecords = SavReader(savFileName)[:5].
You can also do stuff like (numpy required!):
savReader(savFileName)[1:5, 1]"""
is_slice = isinstance(key, slice)
is_array_slice = key is Ellipsis or isinstance(key, tuple)
if is_slice:
start, stop, step = key.indices(self.nCases)
elif is_array_slice:
return self._get_array_slice(key, self.nCases, len(self.header))
else:
key = operator.index(key)
start = key + self.nCases if key < 0 else key
if not 0 <= start < self.nCases:
raise IndexError("Index out of bounds")
stop = start + 1
step = 1
records = self._items(start, stop, step)
if is_slice:
return list(records)
return next(records)
def _get_array_slice(self, key, nRows, nCols):
"""This is a helper function to implement array slicing with numpy"""
if not numpyOk:
raise ImportError("Array slicing requires the numpy library")
is_index = False
rstart = cstart = 0
try:
row, col = key
if row < 0:
row = nRows + row
if col < 0:
col = nCols + col
## ... slices
if isinstance(row, slice) and col is Ellipsis:
# reader[1:2, ...]
rstart, rstop, rstep = row.indices(nRows)
cstart, cstop, cstep = 0, nRows, 1
elif row is Ellipsis and isinstance(col, slice):
# reader[..., 1:2]
rstart, rstop, rstep = 0, nRows, 1
cstart, cstop, cstep = col.indices(nCols)
elif isinstance(row, slice) and isinstance(col, slice):
# reader[1:2, 1:2]
rstart, rstop, rstep = row.indices(nRows)
cstart, cstop, cstep = col.indices(nCols)
elif row is Ellipsis and col is Ellipsis:
# reader[..., ...]
rstart, rstop, rstep = 0, nRows, 1
cstart, cstop, cstep = 0, nCols, 1
## ... indexes
elif isinstance(row, int) and col is Ellipsis:
# reader[1, ...]
rstart, rstop, rstep = row, row + 1, 1
cstart, cstop, cstep = 0, nCols, 1
is_index = True
elif row is Ellipsis and isinstance(col, int):
# reader[..., 1]
rstart, rstop, rstep = 0, nRows, 1
cstart, cstop, cstep = col, col + 1, 1
is_index = True
elif isinstance(row, int) and isinstance(col, int):
# reader[1, 1]
rstart, rstop, rstep = row, row + 1, 1
cstart, cstop, cstep = col, col + 1, 1
is_index = True
# ... slice + index
elif isinstance(row, slice) and isinstance(col, int):
# reader[1:2, 1]
rstart, rstop, rstep = row.indices(nRows)
cstart, cstop, cstep = col, col + 1, 1
elif isinstance(row, int) and isinstance(col, slice):
# reader[1, 1:2]
rstart, rstop, rstep = row, row + 1, 1
cstart, cstop, cstep = col.indices(nCols)
try:
if not 0 <= rstart < nRows:
raise IndexError("Index out of bounds")
if not 0 <= cstart < nCols:
raise IndexError("Index out of bounds")
key = (Ellipsis, slice(cstart, cstop, cstep))
except UnboundLocalError:
msg = "The array index is either invalid, or not implemented"
raise TypeError(msg)
except TypeError:
# reader[...]
rstart, rstop, rstep = 0, nRows, 1
key = (Ellipsis, Ellipsis)
records = self._items(rstart, rstop, rstep)
result = numpy.array(list(records))[key].tolist()
if is_index:
return result[0]
return result
def head(self, n=5):
""" This convenience function returns the first <n> records. """
return self[:abs(n)]
def tail(self, n=5):
""" This convenience function returns the last <n> records. """
return self[-abs(n):]
def __contains__(self, item):
""" This function implements membership testing and returns True if
<idVar> contains <item>. Thus, it requires the 'idVar' parameter to
be set. For example: reader = SavReader(savFileName, idVar="ssn")
"987654321" in reader """
return bool(self.get(item))
def get(self, key, default=None, full=False):
""" This function returns the records for which <idVar> == <key>
if <key> in <savFileName>, else <default>. Thus, the function mimics
dict.get, but note that dict[key] is NOT implemented. NB: Even though
this uses a binary search, this is not very fast on large data (esp.
the first call, and with full=True)
Parameters:
-key: key for which the corresponding record should be returned
-default: value that should be returned if <key> is not found
(default: None)
-full: value that indicates whether *all* records for which
<idVar> == <key> should be returned (default: False)
For example: reader = SavReader(savFileName, idVar="ssn")
reader.get("987654321", "social security number not found!")"""
if not self.idVar in self.varNames:
msg = ("SavReader object must be instantiated with an existing " +
"variable as an idVar argument")
raise NameError(msg)
#two slightly modified functions from the bisect module
def bisect_right(a, x, lo=0, hi=None):
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo + hi) // 2
if x < a[mid][0]:
hi = mid # a[mid][0], not a[mid]
else:
lo = mid + 1
return lo
def bisect_left(a, x, lo=0, hi=None):
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo + hi) // 2
if a[mid][0] < x:
lo = mid + 1 # a[mid][0], not a[mid]
else:
hi = mid
return lo
idPos = self.varNames.index(self.idVar)
if not hasattr(self, "isSorted"):
self.isSorted = True
if self.varTypes[self.idVar] == 0:
if not isinstance(key, (int, float)):
return default
self.recordz = ((record[idPos], i) for i,
record in enumerate(iter(self)))
else:
if not isinstance(key, basestring):
return default
self.recordz = ((record[idPos].rstrip(), i) for i,
record in enumerate(iter(self)))
self.recordz = sorted(self.recordz)
insertLPos = bisect_left(self.recordz, key)
insertRPos = bisect_right(self.recordz, key)
if full:
result = [self[record[1]] for record in
self.recordz[insertLPos: insertRPos]]
else:
if insertLPos == insertRPos:
return default
result = self[self.recordz[insertLPos][1]]
if result:
return result
return default
def getSavFileInfo(self):
""" This function reads and returns some basic information of the open
spss data file."""
return (self.numVars, self.nCases, self.varNames, self.varTypes,
self.formats, self.varLabels, self.valueLabels)
def memoize(f):
"""Memoization decorator
http://code.activestate.com/recipes/577219-minimalistic-memoization/"""
cache = {}
MAXCACHE = 10**4
def memf(*x):
if x not in cache:
cache[x] = f(*x)
if len(cache) > MAXCACHE:
cache.popitem()
return cache[x]
return memf
@memoize
def spss2strDate(self, spssDateValue, fmt, recodeSysmisTo):
"""This function converts internal SPSS dates (number of seconds
since midnight, Oct 14, 1582 (the beginning of the Gregorian calendar))
to a human-readable format"""
try:
if not hasattr(self, "gregorianEpoch"):
self.gregorianEpoch = datetime.datetime(1582, 10, 14, 0, 0, 0)
theDate = (self.gregorianEpoch +
datetime.timedelta(seconds=spssDateValue))
return datetime.datetime.strftime(theDate, fmt)
except OverflowError:
return recodeSysmisTo
except TypeError:
return recodeSysmisTo
except ValueError:
return recodeSysmisTo
def getFileReport(self, savFileName, varNames, varTypes,
formats, nCases):
""" This function prints a report about basic file characteristics """
bytes = os.path.getsize(savFileName)
kb = float(bytes) / 2**10
mb = float(bytes) / 2**20
(fileSize, label) = (mb, "MB") if mb > 1 else (kb, "kB")
systemString = self.systemString
spssVersion = ".".join(map(str, self.spssVersion))
lang, cp = locale.getlocale()
intEnc = "Utf-8/Unicode" if self.ioUtf8 else "Codepage (%s)" % cp
varlist = []
line = " %%0%sd. %%s (%%s - %%s)" % len(str(len(varNames) + 1))
for cnt, varName in enumerate(varNames):
lbl = "string" if varTypes[varName] > 0 else "numerical"
format_ = formats[varName]
varlist.append(line % (cnt + 1, varName, format_, lbl))
info = {"savFileName": savFileName,
"fileSize": fileSize,
"label": label,
"nCases": nCases,
"nCols": len(varNames),
"nValues": nCases * len(varNames),
"spssVersion": "%s (%s)" % (systemString, spssVersion),
"ioLocale": self.ioLocale,
"ioUtf8": intEnc,
"fileEncoding": self.fileEncoding,
"fileCodePage": self.fileCodePage,
"isCompatible": "Yes" if self.isCompatibleEncoding() else "No",
"local_language": lang,
"local_encoding": cp,
"varlist": os.linesep.join(varlist),
"sep": os.linesep,
"asterisks": 70 * "*"}
report = ("%(asterisks)s%(sep)s" +
"*File %(savFileName)r (%(fileSize)3.2f %(label)s) has " +
"%(nCols)s columns (variables) and %(nCases)s rows " +
"(%(nValues)s values)%(sep)s" +
"*The file was created with SPSS version: %(spssVersion)s%" +
"(sep)s" +
"*The interface locale is: %(ioLocale)r%(sep)s" +
"*The interface mode is: %(ioUtf8)s%(sep)s" +
"*The file encoding is: %(fileEncoding)r (Code page: " +
"%(fileCodePage)s)%(sep)s" +
"*File encoding and the interface encoding are compatible:" +
" %(isCompatible)s%(sep)s" +
"*Your computer's locale is: %(local_language)r (Code " +
"page: %(local_encoding)s)%(sep)s" +
"*The file contains the following variables:%(sep)s" +
"%(varlist)s%(sep)s%(asterisks)s%(sep)s")
return report % info
def getHeader(self, selectVars):
"""This function returns the variable names, or a selection thereof
(as specified as a list using the selectVars parameter), as a list."""
if selectVars is None:
header = self.varNames
elif isinstance(selectVars, (list, tuple)):
diff = set(selectVars).difference(set(self.varNames))
if diff:
msg = "Variable names misspecified (%r)" % ", ".join(diff)
raise NameError(msg)
varPos = [varNames.index(v) for v in self.varNames
if v in selectVars]
self.selector = operator.itemgetter(*varPos)
header = self.selector(self.varNames)
header = [header] if not isinstance(header, tuple) else header
else:
msg = ("Variable names list misspecified. Must be 'None' or a " +
"list or tuple of existing variables")
raise TypeError(msg)
return header
class SavWriter(Header):
""" Write Spss system files (.sav, .zsav)
Parameters:
* Formal
-savFileName: the file name of the spss data file. File names that end with
'.zsav' are compressed using the ZLIB (ZSAV) compression scheme
(requires v21 SPSS I/O files), while for file names that end with '.sav'
the 'old' compression scheme is used (it is not possible to generate
uncompressed files unless you modify the source code).
-varNames: list of the variable names in the order in which they appear in
in the spss data file.
-varTypes: varTypes dictionary {varName: varType}, where varType == 0 means
'numeric', and varType > 0 means 'character' of that length (in bytes)
* Optional (the associated SPSS commands are given in CAPS)
-valueLabels: value label dictionary {varName: {value: label}} Cf. VALUE
LABELS (default: None)
-varLabels: variable label dictionary {varName: varLabel}. Cf. VARIABLE
LABEL (default: None)
-formats: format_ dictionary {varName: printFmt} Cf. FORMATS
(default: None)
-missingValues: missing values dictionary {varName: {missing value spec}}
(default: None). Cf. MISSING VALUES. For example:
{'someNumvar1': {'values': [999, -1, -2]}, # discrete values
'someNumvar2': {'lower': -9, 'upper':-1}, # range, cf. -9 THRU -1
'someNumvar3': {'lower': -9, 'upper':-1, 'value': 999},
'someStrvar1': {'values': ['foo', 'bar', 'baz']},
'someStrvar2': {'values': 'bletch'}}
---The following three parameters must all three be set, if used---
-measureLevels: measurement level dictionary {varName: <level>}
Valid levels are: "unknown", "nominal", "ordinal", "scale",
"ratio", "flag", "typeless". Cf. VARIABLE LEVEL (default: None)
-columnWidths: column display width dictionary {varName: <int>}.
Cf. VARIABLE WIDTH. (default: None --> >= 10 [stringVars] or automatic
[numVars])
-alignments: alignment dictionary {varName: <left/center/right>}
Cf. VARIABLE ALIGNMENT (default: None --> left)
---
-varSets: sets dictionary {setName: list_of_valid_varNames}.
Cf. SETSMR command. (default: None)
-varRoles: variable roles dictionary {varName: varRole}, where varRole
may be any of the following: 'both', 'frequency', 'input', 'none',
'partition', 'record ID', 'split', 'target'. Cf. VARIABLE ROLE
(default: None)
-varAttributes: variable attributes dictionary {varName: {attribName:
attribValue} (default: None). For example: varAttributes = {'gender':
{'Binary': 'Yes'}, 'educ': {'DemographicVars': '1'}}. Cf. VARIABLE
ATTRIBUTES. (default: None)
-fileAttributes: file attributes dictionary {attribName: attribValue}
For example: {'RevisionDate[1]': '10/29/2004', 'RevisionDate[2]':
'10/21/2005'}. Square brackets indicate attribute arrays, which must
start with 1. Cf. FILE ATTRIBUTES. (default: None)
-fileLabel: file label string, which defaults to "File created by user
<username> at <datetime>" is file label is None. Cf. FILE LABEL
(default: None)
-multRespDefs: Multiple response sets definitions (dichotomy groups or
category groups) dictionary {setName: <set definition>}. In SPSS syntax,
'setName' has a dollar prefix ('$someSet'). See also docstring of
multRespDefs method. Cf. MRSETS. (default: None)
-caseWeightVar: valid varName that is set as case weight (cf. WEIGHT BY)
-overwrite: Boolean that indicates whether an existing Spss file should be
overwritten (default: True)
-ioUtf8: indicates the mode in which text communicated to or from the
I/O Module will be. Valid values are True (UTF-8/unicode mode, cf. SET
UNICODE=ON) or False (Codepage mode, SET UNICODE=OFF) (default: False)
-ioLocale: indicates the locale of the I/O module, cf. SET LOCALE (default:
None, which is the same as locale.getlocale()[0])
-mode: indicates the mode in which <savFileName> should be opened. Possible
values are "wb" (write), "ab" (append), "cp" (copy: initialize header
using <refSavFileName> as a reference file, cf. APPLY DICTIONARY).
(default: "wb")
-refSavFileName: reference file that should be used to initialize the
header (aka the SPSS data dictionary) containing variable label, value
label, missing value, etc etc definitions. Only relevant in conjunction
with mode="cp". (default: None)
Typical use:
records = [['Test1', 1, 1], ['Test2', 2, 1]]
varNames = ['var1', 'v2', 'v3']
varTypes = {'var1': 5, 'v2': 0, 'v3': 0}
savFileName = "test.sav"
with SavWriter(savFileName, varNames, varTypes) as sav:
sav.writerows(records)
"""
def __init__(self, savFileName, varNames, varTypes, valueLabels=None,
varLabels=None, formats=None, missingValues=None,
measureLevels=None, columnWidths=None, alignments=None,
varSets=None, varRoles=None, varAttributes=None,
fileAttributes=None, fileLabel=None, multRespDefs=None,
caseWeightVar=None, overwrite=True, ioUtf8=False,
ioLocale=None, mode="wb", refSavFileName=None):
""" Constructor. Initializes all vars that can be recycled """
super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
self.savFileName = savFileName
self.varNames = self.encode(varNames)
self.varTypes = self.encode(varTypes)
self.overwrite = overwrite
self.mode = mode
self.refSavFileName = refSavFileName
self.fh = super(Header, self).openSavFile(self.savFileName, self.mode,
self.refSavFileName)
self.myStruct = self.getStruct(self.varTypes, self.varNames, self.mode)
self.pack_into = self.myStruct.pack_into
self.sysmis_ = self.sysmis
self.ioUtf8_ = ioUtf8
self.pad_8_lookup = self._getPaddingLookupTable(self.varTypes)
#self._getVarHandles()
if self.mode == "wb":
self.openWrite(self.savFileName, self.overwrite)
self.varNamesTypes = self.varNames, self.varTypes
self.valueLabels = valueLabels
self.varLabels = varLabels
self.formats = formats
self.missingValues = missingValues
self.measureLevels = measureLevels
self.columnWidths = columnWidths
self.alignments = alignments
self.varSets = varSets
self.varRoles = varRoles
self.varAttributes = varAttributes
self.fileAttributes = fileAttributes
self.fileLabel = fileLabel
self.multRespDefs = multRespDefs
self.caseWeightVar = caseWeightVar
triplet = [measureLevels, columnWidths, alignments]
if all([item is None for item in triplet]):
self._setColWidth10()
self.textInfo = self.savFileName
self.commitHeader()
self.caseBuffer = self.getCaseBuffer()
def __enter__(self):
"""This function returns the writer object itself so the writerow and
writerows methods become available for use with 'with' statements"""
return self
def __exit__(self, type, value, tb):
""" This function closes the spss data file."""
if type is not None:
pass # Exception occurred
self.closeSavFile(self.fh, mode="wb")
#self.closeFile()
def _getVarHandles(self):
"""This function returns a handle for a variable, which can then be
used to read or write (depending on how the file was opened) values
of the variable. If handle is associated with an output file, the
dictionary must be written with spssCommitHeader before variable
handles can be obtained via spssGetVarHandle. Helper function for
__setitem__. *Not currently used*"""
varHandles = {}
varHandle = c_double()
func = self.spssio.spssGetVarHandle
for varName in self.varNames:
retcode = func(c_int(self.fh), c_char_p(varName), byref(varHandle))
varHandles[varName] = varHandle.value
if retcode > 0:
msg = "Error getting variable handle for variable %r"
raise SPSSIOError(msg % varName, retcode)
return varHandles
def __setitem__(self, varName, value):
"""This function sets the value of a variable for the current case.
The current case is not written out to the data file until
spssCommitCaseRecord is called. *Not currently used*, but it was just
begging to be implemented. ;-) Do NOT use in conjunction with
wholeCaseOut. For example: self['someNumVar'] = 10
self['someStrVar'] = 'foo'"""
if not isinstance(value, (float, int, basestring)):
raise ValueError("Value %r has wrong type: %s" %
value, type(value))
varHandle = self.varHandles[varName]
if self.varTypes[varName] == 0:
funcN = self.spssio.spssSetValueNumeric
retcode = funcN(c_int(self.fh), c_double(varHandle),
c_double(value))
else:
funcC = self.spssio.spssSetValueChar
retcode = funcC(c_int(self.fh), c_double(varHandle),
c_char_p(value))
if retcode > 0:
isString = isinstance(value, basestring)
valType = "character" if isString else "numerical"
msg = "Error setting %s value %r for variable %r"
raise SPSSIOError(msg % (valType, value, varName), retcode)
def openWrite(self, savFileName, overwrite):
""" This function opens a file in preparation for creating a new IBM
SPSS Statistics data file"""
if os.path.exists(savFileName) and not os.access(savFileName, os.W_OK):
raise IOError("No write access for file %r" % savFileName)
if overwrite or not os.path.exists(savFileName):
# always compress files, either zsav or standard.
if savFileName.lower().endswith(".zsav"):
self.fileCompression = "zlib" # only with v21 libraries!
else:
self.fileCompression = "standard"
elif not overwrite and os.path.exists(savFileName):
raise IOError("File %r already exists!" % savFileName)
def convertDate(self, day, month, year):
"""This function converts a Gregorian date expressed as day-month-year
to the internal SPSS date format. The time portion of the date variable
is set to 0:00. To set the time portion if the date variable to another
value, use convertTime."""
d, m, y = c_int(day), c_int(month), c_int(year)
spssDate = c_double()
retcode = self.spssio.spssConvertDate(d, m, y, byref(spssDate))
if retcode > 0:
msg = "Problem converting date value '%s-%s-%s'" % (d, m, y)
raise SPSSIOError(msg, retcode)
return spssDate.value
def convertTime(self, day, hour, minute, second):
"""This function converts a time given as day, hours, minutes, and
seconds to the internal SPSS time format."""
d, h, m, s, spssTime = (c_int(day), c_int(hour), c_int(minute),
c_double(float(second)), c_double())
retcode = self.spssio.spssConvertTime(d, h, m, s, byref(spssTime))
if retcode > 0:
msg = ("Problem converting time value '%s %s:%s:%s'" %
(day, hour, minute, second))
raise SPSSIOError(msg, retcode)
return spssTime.value
def spssDateTime(self, datetimeStr="2001-12-08", strptimeFmt="%Y-%m-%d"):
""" This function converts a date/time string into an SPSS date,
using a strptime format."""
dt = time.strptime(datetimeStr, strptimeFmt)
day, month, year = dt.tm_mday, dt.tm_mon, dt.tm_year
hour, minute, second = dt.tm_hour, dt.tm_min, dt.tm_sec
return (self.convertDate(day, month, year) +
self.convertTime(0, hour, minute, second))
def commitHeader(self):
"""This function writes the data dictionary to the data file associated
with file handle 'fh'. Before any case data can be written, the
dictionary must be committed; once the dictionary has been committed,
no further changes can be made to it."""
retcode = self.spssio.spssCommitHeader(c_int(self.fh))
if retcode > 0:
raise SPSSIOError("Problem committing header", retcode)
def _getPaddingLookupTable(self, varTypes):
"""Helper function that returns a lookup table that maps string lengths
to string lengths to the nearest ceiled multiple of 8. For example:
{1:%-8s, 7:%-8s, 9: %-16s, 24: %-24s}. Purpose: Get rid of trailing
null bytes"""
strLengths = varTypes.values()
return dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strLengths])
def writerow(self, record):
if cWriterowOK:
cWriterow(self, record)
return
self.pyWriterow(record)
def pyWriterow(self, record):
""" This function writes one record, which is a Python list."""
float_ = float
for i, value in enumerate(record):
varName = self.varNames[i]
varType = self.varTypes[varName]
if varType == 0:
try:
value = float_(value)
except ValueError:
value = self.sysmis_
except TypeError:
value = self.sysmis_
else:
# Get rid of trailing null bytes --> 7 x faster than 'ljust'
value = self.pad_8_lookup[varType] % value
if self.ioUtf8_:
if isinstance(value, unicode):
value = value.encode("utf-8")
record[i] = value
self.record = record
def writerows(self, records):
""" This function writes all records."""
nCases = len(records)
for case, record in enumerate(records):
self.writerow(record)
self.printPctProgress(case, nCases)
if __name__ == "__main__":
import contextlib
import csv
import collections
import pprint
import cProfile
import pstats
#savFileName = r"C:\Program Files\IBM\SPSS\Statistics\20\Samples\English\Employee data.sav"
## ---------------------------------
## SAV DATA DICTIONARY READER
## ---------------------------------
## ----- Typical use
with SavHeaderReader("employee data.sav") as spssDict:
print str(spssDict) # prints a report of the header information
wholeDict = spssDict.dataDictionary()
## ----- Copy header info from one file to another
savFileName = "employee data.sav"
with SavHeaderReader(savFileName) as spssDict:
wholeDict = spssDict.dataDictionary()
reader = SavReader(savFileName)
writer = SavWriter('test_wholeDict.sav', **wholeDict)
with contextlib.nested(reader, writer) as (r, w):
for line in r:
w.writerow(line)
## ---------------------------------
## SAV READER
## ---------------------------------
## ----- Typical use
savFileName = "employee data.sav"
with SavReader(savFileName, returnHeader=True) as sav:
header = sav.next()
for line in sav:
pass # do stuff
## ----- Explore the data file.
## ... Get some basic file info
reader = SavReader(savFileName, idVar="id")
(numVars, nCases, varNames, varTypes,
formats, varLabels, valueLabels) = reader.getSavFileInfo()
print "The file contains %d records" % len(reader)
print str(reader) # prints a file report
## ... Use indexing, slicing, or array slicing to access records
print "The first six records look like this\n", reader[:6]
print "The first record looks like this\n", reader[0]
print "The last four records look like this\n", reader.tail(4)
print "The first five records look like this\n", reader.head()
if numpyOk:
print "First column:\n", reader[..., 0]
print "Row 4 & 5, first three cols\n", reader[4:6, :3]
## ... Do a binary search for records --> idVar
print reader.get(4, "not found") # gets 1st record where id==4
print reader.get(4, "not found", full=True) # gets all records where id==4
## ... Use a named tuple to get specific values
record = collections.namedtuple("_", " ".join(varNames))(*reader[0])
print record.id, record.educ, record.salary, record.bdate
reader.close()
## ----- Convert file into .csv
savFileName = "employee data.sav"
csvFileName = "test_out.csv"
sav = SavReader(savFileName, ioLocale="dutch") # locale spec depends on OS
f = open(csvFileName, "wb")
with contextlib.nested(sav, f) as (sav, f):
writer = csv.writer(f)
for line in sav:
writer.writerow(line)
print "Done! Csv file written: %s" % f.name
## ---------------------------------
## SAV WRITER
## ---------------------------------
## ----- Write many rows
savFileName = "another_test.zsav" # .zsav = zlib compressed (--> v21 libs)
records = [['Test1', 1, 1, '2010-08-11'], ['Test2', 2, 1, '1910-01-12']]
varNames = ['var1', 'var2', 'var3', 'bdate']
varTypes = {'var1': 50, 'var2': 0, 'var3': 0, 'bdate': 10}
with SavWriter(savFileName, varNames, varTypes) as sav:
sav.writerows(records)
print "Done! %s" % sav.savFileName
savFileName = "employee data_OUT.sav"
varNames = ['id', 'gender', 'bdate', 'educ', 'jobcat', 'salary',
'salbegin', 'jobtime', 'prevexp', 'minority']
varLabels = {'gender': 'guys/gals'}
varTypes = {'salary': 0, 'jobcat': 0, 'bdate': 0, 'minority': 0,
'prevexp': 0, 'gender': 1, 'salbegin': 0, 'jobtime': 0,
'educ': 0, 'id': 0}
varSets = {'SALARY': ['salbegin', 'salary'],
'DEMOGR': ['gender', 'minority', 'educ']}
varAttributes = {'salary': {'DemographicVars': '1'},
'jobcat': {'DemographicVars': '1'},
'gender': {'Binary': 'Yes'},
'educ': {'DemographicVars': '1'}}
fileAttributes = {'TheDate[2]': '10/21/2005',
'RevisionDate[2]': '10/21/2005',
'RevisionDate[1]': '10/29/2004'}
missingValues = {'educ': {'values': [1, 2, 3]}, 'gender': {'values': 'x'}}
records = [[1.0, 'm ', 11654150400.0, 15.0, 3.0,
57000.0, 27000.0, 98.0, 144.0, 0.0],
[2.0, 'm ', 11852956800.0, 16.0, 1.0,
40200.0, 18750.0, 98.0, 36.0, 0.0]]
with SavWriter(savFileName, varNames, varTypes,
varLabels=varLabels, varSets=varSets,
missingValues=missingValues, varAttributes=varAttributes,
fileAttributes=fileAttributes) as sav:
sav.writerows(records)
print "Done! %s" % sav.savFileName
## ----- Write one row
# ... var1 is a 5-char string var, the others are numerical:
# ... formats, varLabels, valueLabels, missingValues etc. may also be
# ... None (default).
records = [['Test1', 1, 1, '2010-08-11'], ['Test2', 2, 1, '1910-01-12']]
varNames = ['var1', 'v2', 'v3', 'bdate']
varTypes = {'var1': 41, 'v2': 0, 'v3': 0, 'bdate': 0}
formats = {'var1': 'A41', 'v2': 'F3.1', 'v3': 'F5.1', 'bdate': 'EDATE40'}
missingValues = {'var1': {'values': ['Test1', 'Test2']},
'v2': {'values': 1}}
varLabels = {'var1': 'This is variable 1',
'v2': 'This is v2!',
'bdate': 'dob'}
valueLabels = {'var1': {'Test1': 'Test1 value label',
'Test2': 'Test2 value label'},
'v2': {1: 'yes', 2: 'no'}}
# This also shows how date fields can be converted into spss dates.
# Spss dates are *stored* as the number of seconds since Oct 14, 1582, but
# *displayed* as <format>. In this case they are displayed as EDATE
# (European date, cf. ADATE = American date), ie. as dd.mm.yyyy
savFileName = "test1.sav"
with SavWriter(savFileName, varNames, varTypes,
valueLabels, varLabels, formats) as sav:
pos = varNames.index("bdate")
for record in records:
record[pos] = sav.spssDateTime(record[pos], "%Y-%m-%d")
sav.writerow(record)
print "Done! %s" % sav.savFileName
## a test to check if non-ascii encodings work well
## source: http://www.omniglot.com/language/phrases/hello.htm
greetings = [
[0, 'Arabic', u'\u0627\u0644\u0633\u0644\u0627\u0645\u0020\u0639\u0644\u064a\u0643\u0645'],
[1, 'Assamese', u'\u09a8\u09ae\u09b8\u09cd\u0995\u09be\u09f0'],
[2, 'Bengali', u'\u0986\u09b8\u09b8\u09be\u09b2\u09be\u09ae\u09c1\u0986\u09b2\u09be\u0987\u0995\u09c1\u09ae'],
[3, 'English', u'Greetings and salutations'],
[4, 'Georgian', u'\u10d2\u10d0\u10db\u10d0\u10e0\u10ef\u10dd\u10d1\u10d0'],
[5, 'Kazakh', u'\u0421\u04d9\u043b\u0435\u043c\u0435\u0442\u0441\u0456\u0437 \u0431\u0435'],
[6, 'Russian', u'\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435'],
[7, 'Spanish', u'\xa1Hola!'],
[8, 'Swiss German', u'Gr\xfcezi'],
[9, 'Thai', u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'],
[10, 'Walloon', u'Bondjo\xfb'],
[11, 'Telugu', u'\u0c0f\u0c2e\u0c02\u0c21\u0c40'],
]
savFileName = "greetings.sav"
varNames = ['line', u'Bondjo\xfb', 'greeting']
varTypes = {'line': 0, u'Bondjo\xfb': 20, 'greeting': 50}
valueLabels = {u'Bondjo\xfb': {'Thai': u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'}}
missingValues = {'line': {'lower': 0, 'upper': 9}, u'Bondjo\xfb': {'values': u'\xa1Hola!'}}
varLabels = {'greeting': u'\u0627\u0644\u0633\u0644\u0627\u0645\u0020\u0639\u0644\u064a\u0643\u0645'}
with SavWriter(savFileName, varNames, varTypes, valueLabels, varLabels,
missingValues=missingValues, ioUtf8=True) as sav:
sav.writerows(greetings)
with SavReader(savFileName, ioUtf8=True)as sav:
for lino, line in enumerate(sav):
print lino, "%s -- %s" % (line[1].strip(), line[2]), line
with SavHeaderReader(savFileName, ioUtf8=True)as spssDict:
wholeDict = spssDict.dataDictionary()
#print wholeDict["varLabels"]["greeting"]
print unicode(spssDict)
pprint.pprint(wholeDict)
Diff to Previous Revision
--- revision 11 2012-12-25 16:07:23
+++ revision 12 2013-02-20 22:07:27
@@ -12,6 +12,10 @@
# ANY FEEDBACK ON THIS CODE IS WELCOME: "@".join(["fomcl", "yahoo.com"])
# Mac support added Oct-22-2011: by Rich Sadowsky
# "@".join(["rich", "richsad.com"])
+
+
+## January 2013:
+# changed implementation of freeMemory method (again!)
## December 2012:
# -Added support for slicing, indexing, array slicing + other special methods
@@ -27,6 +31,14 @@
# -Changed some Reader defaults (verbose=False, returnHeader=False)
# -Renamed SavDataDictionaryReader into SavHeaderReader
##
+
+
+__author__ = "Albert-Jan Roskam" + " " + "@".join(["fomcl", "yahoo.com"])
+__version__ = "3.1.1"
+
+# change this to 'True' in case you experience segmentation
+# faults related to freeing memory.
+segfaults = False
from ctypes import *
import ctypes.util
@@ -43,12 +55,13 @@
import getpass
import encodings
import functools
+import gc
try:
import psyco
- psycoOk = True
+ psycoOk = True # reading 66 % faster
except ImportError:
print ("NOTE. Psyco module not found. Install this module " +
- "to increase program performance")
+ "to increase reader performance")
psycoOk = False
try:
import numpy
@@ -57,13 +70,13 @@
print ("NOTE. Numpy module not found. Install this module " +
"to use array slicing")
numpyOk = False
-
-# change this to 'True' in case you experience segmentation
-# faults related to freeing memory.
-segfaults = False
-
-__author__ = "Albert-Jan Roskam" + " " + "@".join(["fomcl", "yahoo.com"])
-__version__ = "3.1.0"
+try:
+ from cWriterow import cWriterow # writing 66 % faster
+ cWriterowOK = True
+except ImportError:
+ print ("NOTE. cWriterow module not found. Install this module " +
+ "to increase writer performance")
+ cWriterowOK = False
retcodes = {
0: "SPSS_OK",
@@ -208,7 +221,6 @@
"SPSS_THREE_MISSVAL": 3,
"SPSS_MISS_RANGE": -2,
"SPSS_MISS_RANGEANDVAL": -3}
-###############################################################################
class SPSSIOError(Exception):
@@ -225,51 +237,6 @@
code = "unknown code %r" % retcode
msg = "%s [retcode: %r]" % (msg, code)
Exception.__init__(self, msg)
-###############################################################################
-
-
-def wide2utf8(fn):
- """Take a unicode file name string and encode it to a multibyte string
- that Windows can use to represent file names (CP65001, UTF-8)
- http://msdn.microsoft.com/en-us/library/windows/desktop/dd374130"""
-
- from ctypes import wintypes
-
- _CP_UTF8 = 65001
- _CP_ACP = 0 # ANSI
- _LPBOOL = POINTER(c_long)
-
- _wideCharToMultiByte = windll.kernel32.WideCharToMultiByte
- _wideCharToMultiByte.restype = c_int
- _wideCharToMultiByte.argtypes = [wintypes.UINT, wintypes.DWORD,
- wintypes.LPCWSTR, c_int, wintypes.LPSTR, c_int, wintypes.LPCSTR, _LPBOOL]
-
- codePage = _CP_UTF8
- dwFlags = 0
- lpWideCharStr = fn
- cchWideChar = len(fn)
- lpMultiByteStr = None
- cbMultiByte = 0 # zero requests size
- lpDefaultChar = None
- lpUsedDefaultChar = None
-
- # get size
- mbcssize = _wideCharToMultiByte(
- codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
- cbMultiByte, lpDefaultChar, lpUsedDefaultChar)
- if mbcssize <= 0:
- raise WinError(mbcssize)
- lpMultiByteStr = create_string_buffer(mbcssize)
-
- # convert
- retcode = _wideCharToMultiByte(
- codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
- mbcssize, lpDefaultChar, lpUsedDefaultChar)
- if retcode <= 0:
- raise WinError(retcode)
- return lpMultiByteStr.value
-
-###############################################################################
class Generic(object):
@@ -304,7 +271,7 @@
if not isinstance(fn, unicode):
return fn
elif sys.platform.startswith("win"):
- return wide2utf8(fn)
+ return self.wide2utf8(fn)
else:
encoding = sys.getfilesystemencoding()
encoding = "utf-8" if not encoding else encoding # actually, ascii
@@ -348,6 +315,47 @@
raise IOError(msg % (func.__name__, self.savFileName))
return res
+ def wide2utf8(self, fn):
+ """Take a unicode file name string and encode it to a multibyte string
+ that Windows can use to represent file names (CP65001, UTF-8)
+ http://msdn.microsoft.com/en-us/library/windows/desktop/dd374130"""
+
+ from ctypes import wintypes
+
+ _CP_UTF8 = 65001
+ _CP_ACP = 0 # ANSI
+ _LPBOOL = POINTER(c_long)
+
+ _wideCharToMultiByte = windll.kernel32.WideCharToMultiByte
+ _wideCharToMultiByte.restype = c_int
+ _wideCharToMultiByte.argtypes = [wintypes.UINT, wintypes.DWORD,
+ wintypes.LPCWSTR, c_int, wintypes.LPSTR, c_int, wintypes.LPCSTR, _LPBOOL]
+
+ codePage = _CP_UTF8
+ dwFlags = 0
+ lpWideCharStr = fn
+ cchWideChar = len(fn)
+ lpMultiByteStr = None
+ cbMultiByte = 0 # zero requests size
+ lpDefaultChar = None
+ lpUsedDefaultChar = None
+
+ # get size
+ mbcssize = _wideCharToMultiByte(
+ codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
+ cbMultiByte, lpDefaultChar, lpUsedDefaultChar)
+ if mbcssize <= 0:
+ raise WinError(mbcssize)
+ lpMultiByteStr = create_string_buffer(mbcssize)
+
+ # convert
+ retcode = _wideCharToMultiByte(
+ codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
+ mbcssize, lpDefaultChar, lpUsedDefaultChar)
+ if retcode <= 0:
+ raise WinError(retcode)
+ return lpMultiByteStr.value
+
def openSavFile(self, savFileName, mode="rb", refSavFileName=None):
"""This function opens IBM SPSS Statistics data files in mode <mode>
and returns a handle that should be used for subsequent operations on
@@ -399,24 +407,6 @@
if retcode > 0:
raise SPSSIOError("Error closing file in mode %r" % mode, retcode)
- def closeFile(self):
- """Close file descriptor"""
- try:
- if self.fd: # return value is not null
- self.libc.fclose.errcheck = self.errcheck
- self.libc.fclose(c_void_p(self.fd)) # error in Linux?
- else:
- try:
- # Windows
- self.libc._close.errcheck = self.errcheck
- self.libc._close(c_void_p(self.fd))
- except AttributeError:
- # Linux
- self.libc.close.errcheck = self.errcheck
- self.libc.close(c_void_p(self.fd))
- except EnvironmentError, e:
- print e
-
@property
def releaseInfo(self):
"""This function reports release- and machine-specific information
@@ -434,16 +424,44 @@
@property
def spssVersion(self):
- """Return the SPSS version as a three-tuple indicating major,
- minor, and fixpack version as ints. NB: in the transition from
- SPSS to IBM, a new four-digit versioning nomenclature is used.
- This function uses the old three-digit nomenclature. Therefore,
- no patch version information is available."""
+ """Return the SPSS version that was used to create the opened file
+ as a three-tuple indicating major, minor, and fixpack version as
+ ints. NB: in the transition from SPSS to IBM, a new four-digit
+ versioning nomenclature is used. This function returns the old
+ three-digit nomenclature. Therefore, no patch version information
+ is available."""
info = self.releaseInfo
major = info["release number"]
minor = info["release subnumber"]
fixpack = info["fixpack number"]
return major, minor, fixpack
+
+ @property
+ def fileCompression(self):
+ """Get/Set the file compression.
+ Returns/Takes a compression switch which may be any of the following:
+ 'uncompressed', 'standard', or 'zlib'. Zlib comression requires SPSS
+ v21 I/O files."""
+ compression = {0: "uncompressed", 1: "standard", 2: "zlib"}
+ compSwitch = c_int()
+ func = self.spssio.spssGetCompression
+ retcode = func(c_int(self.fh), byref(compSwitch))
+ if retcode > 0:
+ raise SPSSIOError("Error getting file compression", retcode)
+ return compression.get(compSwitch.value)
+
+ @fileCompression.setter
+ def fileCompression(self, compSwitch):
+ compression = {"uncompressed": 0, "standard": 1, "zlib": 2}
+ compSwitch = compression.get(compSwitch)
+ func = self.spssio.spssSetCompression
+ retcode = func(c_int(self.fh), c_int(compSwitch))
+ invalidSwitch = retcodes.get(retcode) == 'SPSS_INVALID_COMPSW'
+ if invalidSwitch and self.spssVersion[0] < 21:
+ msg = "Writing zcompressed files requires >=v21 SPSS I/O libraries"
+ raise ValueError(msg)
+ elif retcode > 0:
+ raise SPSSIOError("Error setting file compression", retcode)
@property
def systemString(self):
@@ -511,8 +529,10 @@
@property
def missingValuesLowHigh(self):
"""This function returns the 'lowest' and 'highest' values used for
- numeric missing value ranges on the host system. It may be called at
- any time."""
+ numeric missing value ranges on the host system. This can be used in
+ a similar way as the LO and HI keywords in missing values
+ specifications (cf. MISSING VALUES foo (LO THRU 0). It may be called
+ at any time."""
lowest, highest = c_double(), c_double()
func = self.spssio.spssLowHighVal
retcode = func(byref(lowest), byref(highest))
@@ -638,7 +658,7 @@
@property
def record(self):
- """Get/Set a whole record from/to a pre-allocated buffer"""
+ """Get/Set a whole record from/to a pre-allocated buffer"""
retcode = self.wholeCaseIn(c_int(self.fh),
byref(self.caseBuffer))
if retcode > 0:
@@ -656,7 +676,8 @@
retcode = self.wholeCaseOut(c_int(self.fh),
c_char_p(self.caseBuffer.raw))
if retcode > 0:
- raise SPSSIOError("Problem writing row:\n" + record, retcode)
+ raise SPSSIOError("Problem writing row:\n" + \
+ unicode(record, "utf-8"), retcode)
def printPctProgress(self, nominator, denominator):
"""This function prints the % progress when reading and writing
@@ -664,7 +685,6 @@
if nominator and nominator % 10**4 == 0:
pctProgress = (float(nominator) / denominator) * 100
print "%2.1f%%... " % pctProgress,
-###############################################################################
class Header(Generic):
@@ -679,8 +699,8 @@
def __init__(self, savFileName, mode, refSavFileName, ioUtf8, ioLocale=None):
"""Constructor"""
super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
- self.freeMemoryItems = {}
self.spssio = self.loadLibrary()
+ self.libc = cdll.LoadLibrary(ctypes.util.find_library("c"))
self.fh = super(Header, self).openSavFile(savFileName, mode,
refSavFileName)
self.varNames, self.varTypes = self.varNamesTypes
@@ -729,35 +749,18 @@
return dict([(utf8dify(x), utf8dify(y)) for x, y in item.items()])
return utf8dify(item)
- def freeMemory(self, items):
+ def freeMemory(self, funcName, *args):
"""Clean up: free memory claimed by e.g. getValueLabels and
getVarNamesTypes"""
- # Previously crashed under other systems than WinXP (eg. Linux & Win7)
- # This function actually has made my receding hairline worse ;-)
- retcodex = []
- isWindows = sys.platform.startswith("win")
- pfVersion = platform.version().split(".")[0]
- pfVersion = int(pfVersion) if pfVersion.isdigit() else 5 # 5 = WinXP
- for funcName, args in sorted(items.iteritems()):
- func = getattr(self.spssio, funcName.split()[0])
- if isWindows and pfVersion <= 5:
- retcode = func(*args)
- retcodex.append(retcode)
- else: # Linux Ubuntu and Windows 7, probably more
- try:
- # free arrays element-by-element
- array1, array2, num = args
- for element1, element2 in zip(array1, array2):
- retcode = func(element1, element2, num)
- retcodex.append(retcode)
- except ValueError:
- retcode = func(*args)
- retcodex.append(retcode)
- okay_or_not = "NOT okay!!!" if retcode > 0 else "okay"
- print "...freeing %s --> %s" % (funcName[8:], okay_or_not)
- retcode = max(retcodex)
- if retcode > 0:
- raise SPSSIOError("Error freeing memory", retcode)
+ gc.collect()
+ if segfaults:
+ return
+ print "... freeing" , funcName[8:]
+ func = getattr(self.spssio, funcName)
+ retcode = func(*args)
+ if retcode > 0:
+ msg = "Error freeing memory using %s" % funcName
+ raise SPSSIOError(msg, retcode)
@property
def numberofCases(self):
@@ -795,21 +798,30 @@
indicating a string variable of that size (in bytes)."""
if hasattr(self, "varNames"):
return self.varNames, self.varTypes
+
+ # initialize arrays
numVars = self.numberofVariables
numVars_ = c_int(numVars)
- varNamesArr = (POINTER(c_char_p) * numVars)()
- varTypesArr = (POINTER(c_int) * numVars)()
+ varNamesArr = (POINTER(c_char_p * numVars))()
+ varTypesArr = (POINTER(c_int * numVars))()
+
+ # get variable names
func = self.spssio.spssGetVarNames
retcode = func(c_int(self.fh), byref(numVars_),
byref(varNamesArr), byref(varTypesArr))
if retcode > 0:
raise SPSSIOError("Error getting variable names & types", retcode)
+
+ # get array contents
varNames = [varNamesArr[0][i] for i in xrange(numVars)]
varTypes = [varTypesArr[0][i] for i in xrange(numVars)]
if self.ioUtf8:
varNames = [varName.decode("utf-8") for varName in varNames]
- funcStr = "spssFreeVarNames"
- self.freeMemoryItems[funcStr] = (varNamesArr, varTypesArr, numVars)
+
+ # clean up
+ args = (varNamesArr, varTypesArr, numVars)
+ self.freeMemory("spssFreeVarNames", *args)
+
return varNames, dict(zip(varNames, varTypes))
@varNamesTypes.setter
@@ -846,32 +858,58 @@
Takes a dictionary of the form {varName: {value: valueLabel}:
--{'numGender': {1: 'female', {2: 'male'}}
--{'strGender': {'f': 'female', 'm': 'male'}}"""
- MAX_ARRAY_SIZE = 1000 # assume never more than 1000 labels
- numLabels, valueLabels = c_int(), {}
+ def initArrays(isNumeric=True, size=1000):
+ """default size assumes never more than 1000 labels"""
+ labelsArr = (POINTER(c_char_p * size))()
+ if isNumeric:
+ return (POINTER(c_double * size))(), labelsArr
+ return (POINTER(c_char_p * size))(), labelsArr
+
+ valueLabels = {}
for varName in self.varNames:
- labelsArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
+ vName = self.vNames[varName]
+ numLabels = c_int()
+
+ # step 1: get array size (numeric values)
if self.varTypes[varName] == 0:
- valuesArr = (POINTER(c_double) * MAX_ARRAY_SIZE)()
+ valuesArr, labelsArr = initArrays(True)
func = self.spssio.spssGetVarNValueLabels
- funcStr = "spssFreeVarNValueLabels"
- items = (valuesArr, labelsArr, numLabels)
- self.freeMemoryItems[funcStr] = items
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(valuesArr), byref(labelsArr),
+ byref(numLabels))
+ valuesArr, labelsArr = initArrays(True, numLabels.value)
+
+ # step 1: get array size (string values)
else:
- valuesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
+ valuesArr, labelsArr = initArrays(False)
func = self.spssio.spssGetVarCValueLabels
- funcStr = "spssFreeVarCValueLabels"
- items = (valuesArr, labelsArr, numLabels)
- self.freeMemoryItems[funcStr] = items
- vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(valuesArr), byref(labelsArr),
+ byref(numLabels))
+ valuesArr, labelsArr = initArrays(False, numLabels.value)
+
+ # step 2: get labels with array of proper size
retcode = func(c_int(self.fh), c_char_p(vName), byref(valuesArr),
byref(labelsArr), byref(numLabels))
if retcode > 0:
msg = "Error getting value labels of variable %r"
raise SPSSIOError(msg % varName, retcode)
+
+ # get array contents
+ if not numLabels.value:
+ continue
values = [valuesArr[0][i] for i in xrange(numLabels.value)]
labels = [labelsArr[0][i] for i in xrange(numLabels.value)]
valueLabelsX = [(val, lbl) for val, lbl in zip(values, labels)]
valueLabels[varName] = dict(valueLabelsX)
+
+ # clean up
+ args = (valuesArr, labelsArr, numLabels)
+ if self.varTypes[varName] == 0:
+ self.freeMemory("spssFreeVarNValueLabels", *args)
+ else:
+ self.freeMemory("spssFreeVarCValueLabels", *args)
+
return valueLabels
@valueLabels.setter
@@ -970,7 +1008,7 @@
regex = re.compile("\w+(?P<varWid>\d+)[.]?\d?", re.I)
bareformats, varWids = {}, {}
for varName, format_ in self.formats.iteritems():
- bareformats[varName] = re.sub(r"\d+.?", "", format_)
+ bareformats[varName] = re.sub(r"\d+.", "", format_)
varWids[varName] = int(regex.search(format_).group("varWid"))
return bareformats, varWids
@@ -1012,7 +1050,7 @@
args = (c_int(self.fh), c_char_p(varName), c_int(printFormat),
c_int(printDec), c_int(printWid))
- retcode1, retcode2 = funcP(*args), funcW(*args)
+ retcode1, retcode2 = funcP(*args), funcW(*args)
if retcodes.get(retcode1) == "SPSS_INVALID_PRFOR":
# invalid PRint FORmat
msg = "format_ for %r misspecified (%r)"
@@ -1303,14 +1341,17 @@
if retcode > 0:
msg = "Problem getting variable set information"
raise SPSSIOError(msg, retcode)
- elif retcode == 0:
- varSets_ = {}
- for varSet in varSets.value.split("\n")[:-1]:
- k, v = varSet.split("= ")
- varSets_[k] = v.split()
- self.freeMemoryItems["spssFreeVariableSets"] = (varSets,)
- return varSets_
- return {}
+ if not varSets.value:
+ return {}
+ varSets_ = {}
+ for varSet in varSets.value.split("\n")[:-1]:
+ k, v = varSet.split("= ")
+ varSets_[k] = v.split()
+
+ # clean up
+ self.freeMemory("spssFreeVariableSets", varSets)
+
+ return varSets_
@varSets.setter
def varSets(self, varSets):
@@ -1369,23 +1410,49 @@
{'var1': {'attr name x': 'attr value x','attr name y': 'attr value y'},
'var2': {'attr name a': 'attr value a','attr name b': 'attr value b'}}
"""
- numVars = len(self.varNames)
- attrNamesArr = (POINTER(c_char_p) * numVars)()
- attrValuesArr = (POINTER(c_char_p) * numVars)()
- nAttr, attributes = c_int(), {}
+ # abbreviation for readability and speed
func = self.spssio.spssGetVarAttributes
+
+ # initialize arrays
+ MAX_ARRAY_SIZE = 1000
+ attrNamesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
+ attrValuesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
+
+ attributes = {}
for varName in self.varNames:
vName = self.vNames[varName]
+
+ # step 1: get array size
+ nAttr = c_int()
retcode = func(c_int(self.fh), c_char_p(vName),
byref(attrNamesArr), byref(attrValuesArr),
byref(nAttr))
if retcode > 0:
- msg = "Problem getting variable attributes for variable %r"
+ msg = "@Problem getting attributes of variable %r (step 1)"
raise SPSSIOError(msg % varName, retcode)
+
+ # step 2: get attributes with arrays of proper size
+ nAttr = c_int(nAttr.value)
+ attrNamesArr = (POINTER(c_char_p * nAttr.value))()
+ attrValuesArr = (POINTER(c_char_p * nAttr.value))()
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(attrNamesArr), byref(attrValuesArr),
+ byref(nAttr))
+ if retcode > 0:
+ msg = "Problem getting attributes of variable %r (step 2)"
+ raise SPSSIOError(msg % varName, retcode)
+
+ # get array contents
+ if not nAttr.value:
+ continue
k, v, n = attrNamesArr[0], attrValuesArr[0], nAttr.value
- attributes[varName] = dict([(k[i], v[i]) for i in xrange(n)])
- funcStr = "spssFreeAttributes varAttributes"
- self.freeMemoryItems[funcStr] = (attrNamesArr, attrValuesArr, nAttr)
+ attribute = dict([(k[i], v[i]) for i in xrange(n)])
+ attributes[varName] = attribute
+
+ # clean up
+ args = (attrNamesArr, attrValuesArr, nAttr)
+ self.freeMemory("spssFreeAttributes", *args)
+
return attributes
@varAttributes.setter
@@ -1415,19 +1482,36 @@
{'attrName[1]': 'attrValue1', 'revision[1]': '2010-10-09',
'revision[2]': '2010-10-22', 'revision[3]': '2010-11-19'}
"""
- MAX_ARRAY_SIZE = 100
+ # abbreviation for readability
+ func = self.spssio.spssGetFileAttributes
+
+ # step 1: get array size
+ MAX_ARRAY_SIZE = 100 # assume never more than 100 file attributes
+ attrNamesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
+ attrValuesArr = (POINTER(c_char_p * MAX_ARRAY_SIZE))()
nAttr = c_int()
- attrNamesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
- attrValuesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
- func = self.spssio.spssGetFileAttributes
retcode = func(c_int(self.fh), byref(attrNamesArr),
byref(attrValuesArr), byref(nAttr))
+
+ # step 2: get attributes with arrays of proper size
+ nAttr = c_int(nAttr.value)
+ attrNamesArr = (POINTER(c_char_p * nAttr.value))()
+ attrValuesArr = (POINTER(c_char_p * nAttr.value))()
+ retcode = func(c_int(self.fh), byref(attrNamesArr),
+ byref(attrValuesArr), byref(nAttr))
if retcode > 0:
raise SPSSIOError("Problem getting file attributes", retcode)
+
+ # get array contents
+ if not nAttr.value:
+ return {}
k, v = attrNamesArr[0], attrValuesArr[0]
attributes = dict([(k[i], v[i]) for i in xrange(nAttr.value)])
- funcStr = "spssFreeAttributes fileAttributes"
- self.freeMemoryItems[funcStr] = (attrNamesArr, attrValuesArr, nAttr)
+
+ # clean up
+ args = (attrNamesArr, attrValuesArr, nAttr)
+ self.freeMemory("spssFreeAttributes", *args)
+
return attributes
@fileAttributes.setter
@@ -1460,20 +1544,17 @@
is 'D', the multiple response definition also includes '"countedValue":
countedValue'"""
regex = "\$(?P<setName>\w+)=(?P<setType>[CD])\n?"
- regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
m = re.search(regex + ".*", mrDef, re.I | re.U)
if not m:
return {}
setType = m.group("setType")
if setType == "C": # multiple category sets
regex += " (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?"
- regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
matches = re.findall(regex, mrDef, re.I)
setName, setType, lblLen, lblVarNames = matches[0]
else: # multiple dichotomy sets
regex += ("(?P<valueLen>\d+) (?P<countedValue>\w+)" +
" (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?")
- regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
matches = re.findall(regex, mrDef, re.I | re.U)
setName, setType, valueLen = matches[0][:3]
countedValue, lblLen, lblVarNames = matches[0][3:]
@@ -1492,17 +1573,18 @@
dictionary, into a string that the IO module can use"""
mrespDefs = []
for setName, rest in multRespDefs.iteritems():
+ print setName, rest, rest["setType"]
if rest["setType"] not in ("C", "D"):
- return
+ continue
rest["setName"] = setName
mrespDef = "$%(setName)s=%(setType)s" % rest
- lblLen = len(mrespDef["label"])
+ lblLen = len(rest["label"])
rest["lblLen"] = lblLen
rest["varNames"] = " ".join(rest["varNames"])
tail = "%(varNames)s" if lblLen == 0 else "%(label)s %(varNames)s"
- if mrespDef["setType"] == "C": # multiple category sets
+ if rest["setType"] == "C": # multiple category sets
template = " %%(lblLen)s %s " % tail
- else: # multiple dichotomy sets
+ else: # multiple dichotomy sets
template = " %%(countedValue)s %%(lblLen)s %s " % tail
mrespDef += template % rest
mrespDefs.append(mrespDef)
@@ -1515,7 +1597,6 @@
regex = ("\$(?P<setName>\w+)=(?P<setType>E) (?P<flag1>1)" +
"(?P<flag2>1)? (?P<valueLen>[0-9]+) (?P<countedValue>\w+) " +
"(?P<lblLen>[0-9]+) (?P<lblVarNames>[\w ]+)")
- regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
matches = re.findall(regex, mrDef, re.I | re.U)
setName, setType, flag1, flag2 = matches[0][:4]
valueLen, countedValue, lblLen, lblVarNames = matches[0][4:]
@@ -1531,13 +1612,12 @@
mrDefs = []
for setName, rest in multRespDefs.iteritems():
if rest["setType"] != "E":
- return {}
+ continue
rest["setName"] = setName
v = int(rest["firstVarIsLabel"])
rest["firstVarIsLabel"] = v if v == 1 else ""
rest["valueLen"] = len(rest["countedValue"])
rest["lblLen"] = len(rest["label"])
- rest["label"] = rest["label"]
rest["varNames"] = " ".join(rest["varNames"])
mrDef = "$%(setName)s=%(setType)s 1%(firstVarIsLabel)s "
mrDef += "%(valueLen)s %(countedValue)s %(lblLen)s %(label)s "
@@ -1575,36 +1655,36 @@
"testSetD2": dichotomous2, "testSetEx1": extended1,
"testSetEx2": extended2}
"""
- if self.spssVersion[0] == 21:
- print ("WARNING: getting multiple response definitions may not " +
- "work with this IO version")
+
+ ## Normal Multiple response definitions
+ func = self.spssio.spssGetMultRespDefs
mrDefs = c_char_p()
- func = self.spssio.spssGetMultRespDefs
- retcode = func(c_int(self.fh), byref(mrDefs))
+ retcode = func(c_int(self.fh), pointer(mrDefs))
if retcode > 0:
msg = "Problem getting multiple response definitions"
raise SPSSIOError(msg, retcode)
+
multRespDefs = {}
if mrDefs.value:
for mrDef in mrDefs.value.split("\n"):
for setName, rest in self._getMultRespDef(mrDef).iteritems():
multRespDefs[setName] = rest
- self.freeMemoryItems["spssFreeMultRespDefs Normal"] = (mrDefs,)
-
- # I am not sure whether 'extended' MR definitions complement or replace
- # 'normal' MR definitions. I assumed 'complement'.
+ self.freeMemory("spssFreeMultRespDefs", mrDefs)
+
+ ## Extended Multiple response definitions
mrDefsEx = c_char_p()
func = self.spssio.spssGetMultRespDefsEx
- retcode = func(c_int(self.fh), byref(mrDefsEx))
+ retcode = func(c_int(self.fh), pointer(mrDefsEx))
if retcode > 0:
msg = "Problem getting extended multiple response definitions"
raise SPSSIOError(msg, retcode)
+
multRespDefsEx = {}
if mrDefsEx.value:
for mrDefEx in mrDefsEx.value.split("\n"):
for setName, rest in self._getMultRespDef(mrDefEx).iteritems():
multRespDefsEx[setName] = rest
- self.freeMemoryItems["spssFreeMultRespDefs Extended"] = (mrDefsEx,)
+ self.freeMemory("spssFreeMultRespDefs", mrDefsEx)
multRespDefs.update(multRespDefsEx)
return multRespDefs
@@ -1653,24 +1733,36 @@
@property
@decode
- def dateVariables(self): # entirely untested!
+ def dateVariables(self): # seems to be okay
"""Get/Set DATE information. This function reports the Forecasting
(Trends) date variable information, if any, in IBM SPSS Statistics
data files. Entirely untested and not implemented in reader/writer"""
+ # step 1: get array size
nElements = c_int()
- MAX_ARRAY_SIZE = 100 # or self.nCases?
- dateInfoArr = (POINTER(c_long) * MAX_ARRAY_SIZE)()
func = self.spssio.spssGetDateVariables
+ MAX_ARRAY_SIZE = 100
+ dateInfoArr = (POINTER(c_long * MAX_ARRAY_SIZE))()
retcode = func(c_int(self.fh), byref(nElements), byref(dateInfoArr))
+
+ # step 2: get date info with array of proper size
+ dateInfoArr = (POINTER(c_long * nElements.value))()
+ retcode = func(c_int(self.fh), byref(nElements), byref(dateInfoArr))
if retcode > 0:
raise SPSSIOError("Error getting TRENDS information", retcode)
+
+ # get array contents
nElem = nElements.value
+ if not nElem:
+ return {}
dateInfo = [dateInfoArr[0][i] for i in xrange(nElem)]
fixedDateInfo = dateInfo[:6]
otherDateInfo = [dateInfo[i: i + 3] for i in xrange(6, nElem, 3)]
- self.freeMemoryItems["spssFreeDateVariables"] = (dateInfoArr,)
dateInfo = {"fixedDateInfo": fixedDateInfo,
"otherDateInfo": otherDateInfo}
+
+ # clean up
+ self.freeMemory("spssFreeDateVariables", dateInfoArr)
+
return dateInfo
@dateVariables.setter
@@ -1716,33 +1808,6 @@
raise SPSSIOError("Error setting textInfo", retcode)
@property
- def fileCompression(self):
- """Get/Set the file compression.
- Returns/Takes a compression switch which may be any of the following:
- 'uncompressed', 'standard', or 'zlib'. Zlib comression requires SPSS
- v21 I/O files."""
- compression = {0: "uncompressed", 1: "standard", 2: "zlib"}
- compSwitch = c_int()
- func = self.spssio.spssGetCompression
- retcode = func(c_int(self.fh), byref(compSwitch))
- if retcode > 0:
- raise SPSSIOError("Error getting file compression", retcode)
- return compression.get(compSwitch.value)
-
- @fileCompression.setter
- def fileCompression(self, compSwitch):
- compression = {"uncompressed": 0, "standard": 1, "zlib": 2}
- compSwitch = compression.get(compSwitch)
- func = self.spssio.spssSetCompression
- retcode = func(c_int(self.fh), c_int(compSwitch))
- invalidSwitch = retcodes.get(retcode) == 'SPSS_INVALID_COMPSW'
- if invalidSwitch and self.spssVersion[0] < 21:
- msg = "Writing zcompressed files requires >=v21 SPSS I/O libraries"
- raise ValueError(msg)
- elif retcode > 0:
- raise SPSSIOError("Error setting file compression", retcode)
-
- @property
@decode
def fileLabel(self):
"""Get/Set FILE LABEL (id string)
@@ -1764,7 +1829,6 @@
retcode = self.spssio.spssSetIdString(c_int(self.fh), c_char_p(idStr))
if retcode > 0:
raise SPSSIOError("Error setting file label (id string)", retcode)
-###############################################################################
class SavHeaderReader(Header):
@@ -1818,9 +1882,7 @@
def close(self):
"""This function closes the spss data file and does some cleaning."""
if not segfaults:
- self.freeMemory(self.freeMemoryItems)
self.closeSavFile(self.fh, mode="rb")
- self.closeFile() # this in particular may cause errors in Linux
def dataDictionary(self):
""" This function returns all the dictionary items. It returns
@@ -1870,11 +1932,10 @@
for varName in allValues:
report.append(varName)
return os.linesep.join(report)
-###############################################################################
class SavReader(Header):
- """ Read Spss system files (.sav)
+ """ Read Spss system files (.sav, .zsav)
Parameters:
-savFileName: the file name of the spss data file
@@ -1957,9 +2018,7 @@
def close(self):
"""This function closes the spss data file and does some cleaning."""
if not segfaults:
- self.freeMemory(self.freeMemoryItems)
self.closeSavFile(self.fh, mode="rb")
- self.closeFile()
del self.spssio
def __len__(self):
@@ -2388,12 +2447,11 @@
"list or tuple of existing variables")
raise TypeError(msg)
return header
-###############################################################################
class SavWriter(Header):
- """ Write Spss system files (.sav)
+ """ Write Spss system files (.sav, .zsav)
Parameters:
* Formal
@@ -2433,8 +2491,10 @@
---
-varSets: sets dictionary {setName: list_of_valid_varNames}.
Cf. SETSMR command. (default: None)
- -varRoles: variable roles dictionary {setName: list_of_valid_varNames}.
- Cf. VARIABLE ROLE (default: None)
+ -varRoles: variable roles dictionary {varName: varRole}, where varRole
+ may be any of the following: 'both', 'frequency', 'input', 'none',
+ 'partition', 'record ID', 'split', 'target'. Cf. VARIABLE ROLE
+ (default: None)
-varAttributes: variable attributes dictionary {varName: {attribName:
attribValue} (default: None). For example: varAttributes = {'gender':
{'Binary': 'Yes'}, 'educ': {'DemographicVars': '1'}}. Cf. VARIABLE
@@ -2536,7 +2596,7 @@
if type is not None:
pass # Exception occurred
self.closeSavFile(self.fh, mode="wb")
- self.closeFile()
+ #self.closeFile()
def _getVarHandles(self):
"""This function returns a handle for a variable, which can then be
@@ -2560,9 +2620,9 @@
"""This function sets the value of a variable for the current case.
The current case is not written out to the data file until
spssCommitCaseRecord is called. *Not currently used*, but it was just
- begging to be implemented. ;-) For example:
- self['someNumVar'] = 10
- self['someStrVar'] = 'foo'"""
+ begging to be implemented. ;-) Do NOT use in conjunction with
+ wholeCaseOut. For example: self['someNumVar'] = 10
+ self['someStrVar'] = 'foo'"""
if not isinstance(value, (float, int, basestring)):
raise ValueError("Value %r has wrong type: %s" %
value, type(value))
@@ -2644,11 +2704,16 @@
to string lengths to the nearest ceiled multiple of 8. For example:
{1:%-8s, 7:%-8s, 9: %-16s, 24: %-24s}. Purpose: Get rid of trailing
null bytes"""
- strLenghts = varTypes.values()
- strRange = xrange(min(strLenghts), max(strLenghts) + 1)
- return dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strRange])
+ strLengths = varTypes.values()
+ return dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strLengths])
def writerow(self, record):
+ if cWriterowOK:
+ cWriterow(self, record)
+ return
+ self.pyWriterow(record)
+
+ def pyWriterow(self, record):
""" This function writes one record, which is a Python list."""
float_ = float
for i, value in enumerate(record):
@@ -2659,6 +2724,8 @@
value = float_(value)
except ValueError:
value = self.sysmis_
+ except TypeError:
+ value = self.sysmis_
else:
# Get rid of trailing null bytes --> 7 x faster than 'ljust'
value = self.pad_8_lookup[varType] % value
@@ -2674,8 +2741,6 @@
for case, record in enumerate(records):
self.writerow(record)
self.printPctProgress(case, nCases)
-
-###############################################################################
if __name__ == "__main__":
@@ -2731,7 +2796,7 @@
print "The first record looks like this\n", reader[0]
print "The last four records look like this\n", reader.tail(4)
print "The first five records look like this\n", reader.head()
- if numpyOK:
+ if numpyOk:
print "First column:\n", reader[..., 0]
print "Row 4 & 5, first three cols\n", reader[4:6, :3]
@@ -2746,7 +2811,7 @@
reader.close()
## ----- Convert file into .csv
- savFileName = "in_unicode.sav"
+ savFileName = "employee data.sav"
csvFileName = "test_out.csv"
sav = SavReader(savFileName, ioLocale="dutch") # locale spec depends on OS
f = open(csvFileName, "wb")
@@ -2842,7 +2907,7 @@
[10, 'Walloon', u'Bondjo\xfb'],
[11, 'Telugu', u'\u0c0f\u0c2e\u0c02\u0c21\u0c40'],
]
- savFileName = "greetings_" + greetings[0][-1] + ".sav"
+ savFileName = "greetings.sav"
varNames = ['line', u'Bondjo\xfb', 'greeting']
varTypes = {'line': 0, u'Bondjo\xfb': 20, 'greeting': 50}
valueLabels = {u'Bondjo\xfb': {'Thai': u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'}}
