#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
SavReaderWriter.py: A cross-platform Python interface to the IBM SPSS
Statistics Input Output Module. Read or Write SPSS system files (.sav, .zsav)
"""
# libspssdio.so.1, libspssdio.dylib, spssio32.dll + associated libaries and
# documentation can be downloaded here:
# https://www.ibm.com/developerworks/mydeveloperworks/wikis/home/wiki/We70df3195ec8_4f95_9773_42e448fa9029/page/Downloads%20for%20IBM%C2%AE%20SPSS%C2%AE%20Statistics?lang=en
# ANY FEEDBACK ON THIS CODE IS WELCOME: "@".join(["fomcl", "yahoo.com"])
# Mac support added Oct-22-2011: by Rich Sadowsky
# "@".join(["rich", "richsad.com"])
## December 2012:
# -Added support for slicing, indexing, array slicing + other special methods
# -Added support for writing spss date fields
# -Added support for almost all meta data (missing values, sets, roles, etc.)
# -Added support for 64 bit Windows (tested with Win7) and other OSs
# (z/Linux, Solaris, HP Linux, IBM AIX (untested though)
# -Added support for reading and writing zlib compressed (.zsav) files
# -Removed pesky segfault error when freeing memory
# -Removed errors related to encoding
# -Removed error that prevented files originating from big-endian systems
# being read properly in little-endian systems (and vice versa)
# -Changed some Reader defaults (verbose=False, returnHeader=False)
# -Renamed SavDataDictionaryReader into SavHeaderReader
##
from ctypes import *
import ctypes.util
import struct
import sys
import platform
import os
import re
import operator
import math
import locale
import datetime
import time
import getpass
import encodings
import functools
try:
import psyco
psycoOk = True
except ImportError:
print ("NOTE. Psyco module not found. Install this module " +
"to increase program performance")
psycoOk = False
try:
import numpy
numpyOk = True
except ImportError:
print ("NOTE. Numpy module not found. Install this module " +
"to use array slicing")
numpyOk = False
# change this to 'True' in case you experience segmentation
# faults related to freeing memory.
segfaults = False
__author__ = "Albert-Jan Roskam" + " " + "@".join(["fomcl", "yahoo.com"])
__version__ = "3.1.0"
retcodes = {
0: "SPSS_OK",
1: "SPSS_FILE_OERROR",
2: "SPSS_FILE_WERROR",
3: "SPSS_FILE_RERROR",
4: "SPSS_FITAB_FULL",
5: "SPSS_INVALID_HANDLE",
6: "SPSS_INVALID_FILE",
7: "SPSS_NO_MEMORY",
8: "SPSS_OPEN_RDMODE",
9: "SPSS_OPEN_WRMODE",
10: "SPSS_INVALID_VARNAME",
11: "SPSS_DICT_EMPTY",
12: "SPSS_VAR_NOTFOUND",
13: "SPSS_DUP_VAR",
14: "SPSS_NUME_EXP",
15: "SPSS_STR_EXP",
16: "SPSS_SHORTSTR_EXP",
17: "SPSS_INVALID_VARTYPE",
18: "SPSS_INVALID_MISSFOR",
19: "SPSS_INVALID_COMPSW",
20: "SPSS_INVALID_PRFOR",
21: "SPSS_INVALID_WRFOR",
22: "SPSS_INVALID_DATE",
23: "SPSS_INVALID_TIME",
24: "SPSS_NO_VARIABLES",
27: "SPSS_DUP_VALUE",
28: "SPSS_INVALID_CASEWGT",
30: "SPSS_DICT_COMMIT",
31: "SPSS_DICT_NOTCOMMIT",
33: "SPSS_NO_TYPE2",
41: "SPSS_NO_TYPE73",
45: "SPSS_INVALID_DATEINFO",
46: "SPSS_NO_TYPE999",
47: "SPSS_EXC_STRVALUE",
48: "SPSS_CANNOT_FREE",
49: "SPSS_BUFFER_SHORT",
50: "SPSS_INVALID_CASE",
51: "SPSS_INTERNAL_VLABS",
52: "SPSS_INCOMPAT_APPEND",
53: "SPSS_INTERNAL_D_A",
54: "SPSS_FILE_BADTEMP",
55: "SPSS_DEW_NOFIRST",
56: "SPSS_INVALID_MEASURELEVEL",
57: "SPSS_INVALID_7SUBTYPE",
58: "SPSS_INVALID_VARHANDLE",
59: "SPSS_INVALID_ENCODING",
60: "SPSS_FILES_OPEN",
70: "SPSS_INVALID_MRSETDEF",
71: "SPSS_INVALID_MRSETNAME",
72: "SPSS_DUP_MRSETNAME",
73: "SPSS_BAD_EXTENSION",
74: "SPSS_INVALID_EXTENDEDSTRING",
75: "SPSS_INVALID_ATTRNAME",
76: "SPSS_INVALID_ATTRDEF",
77: "SPSS_INVALID_MRSETINDEX",
78: "SPSS_INVALID_VARSETDEF",
79: "SPSS_INVALID_ROLE",
-15: "SPSS_EMPTY_DEW",
-14: "SPSS_NO_DEW",
-13: "SPSS_EMPTY_MULTRESP",
-12: "SPSS_NO_MULTRESP",
-11: "SPSS_NO_DATEINFO",
-10: "SPSS_NO_CASEWGT",
-9: "SPSS_NO_LABEL",
-8: "SPSS_NO_LABELS",
-7: "SPSS_EMPTY_VARSETS",
-6: "SPSS_NO_VARSETS",
-5: "SPSS_FILE_END",
-4: "SPSS_EXC_VALLABEL",
-3: "SPSS_EXC_LEN120",
-2: "SPSS_EXC_VARLABEL",
-1: "SPSS_EXC_LEN64"}
allFormats = {
1: ("SPSS_FMT_A", "Alphanumeric"),
2: ("SPSS_FMT_AHEX", "Alphanumeric hexadecimal"),
3: ("SPSS_FMT_COMMA", "F Format with commas"),
4: ("SPSS_FMT_DOLLAR", "Commas and floating dollar sign"),
5: ("SPSS_FMT_F", "Default Numeric Format"),
6: ("SPSS_FMT_IB", "Integer binary"),
7: ("SPSS_FMT_PIBHEX", "Positive integer binary - hex"),
8: ("SPSS_FMT_P", "Packed decimal"),
9: ("SPSS_FMT_PIB", "Positive integer binary unsigned"),
10: ("SPSS_FMT_PK", "Positive integer binary unsigned"),
11: ("SPSS_FMT_RB", "Floating point binary"),
12: ("SPSS_FMT_RBHEX", "Floating point binary hex"),
15: ("SPSS_FMT_Z", "Zoned decimal"),
16: ("SPSS_FMT_N", "N Format- unsigned with leading 0s"),
17: ("SPSS_FMT_E", "E Format- with explicit power of 10"),
20: ("SPSS_FMT_DATE", "Date format dd-mmm-yyyy"),
21: ("SPSS_FMT_TIME", "Time format hh:mm:ss.s"),
22: ("SPSS_FMT_DATE_TIME", "Date and Time"),
23: ("SPSS_FMT_ADATE", "Date format dd-mmm-yyyy"),
24: ("SPSS_FMT_JDATE", "Julian date - yyyyddd"),
25: ("SPSS_FMT_DTIME", "Date-time dd hh:mm:ss.s"),
26: ("SPSS_FMT_WKDAY", "Day of the week"),
27: ("SPSS_FMT_MONTH", "Month"),
28: ("SPSS_FMT_MOYR", "mmm yyyy"),
29: ("SPSS_FMT_QYR", "q Q yyyy"),
30: ("SPSS_FMT_WKYR", "ww WK yyyy"),
31: ("SPSS_FMT_PCT", "Percent - F followed by %"),
32: ("SPSS_FMT_DOT", "Like COMMA, switching dot for comma"),
33: ("SPSS_FMT_CCA", "User Programmable currency format"),
34: ("SPSS_FMT_CCB", "User Programmable currency format"),
35: ("SPSS_FMT_CCC", "User Programmable currency format"),
36: ("SPSS_FMT_CCD", "User Programmable currency format"),
37: ("SPSS_FMT_CCE", "User Programmable currency format"),
38: ("SPSS_FMT_EDATE", "Date in dd/mm/yyyy style"),
39: ("SPSS_FMT_SDATE", "Date in yyyy/mm/dd style")}
MAXLENGTHS = {
"SPSS_MAX_VARNAME": (64, "Variable name"),
"SPSS_MAX_SHORTVARNAME": (8, "Short (compatibility) variable name"),
"SPSS_MAX_SHORTSTRING": (8, "Short string variable"),
"SPSS_MAX_IDSTRING": (64, "File label string"),
"SPSS_MAX_LONGSTRING": (32767, "Long string variable"),
"SPSS_MAX_VALLABEL": (120, "Value label"),
"SPSS_MAX_VARLABEL": (256, "Variable label"),
"SPSS_MAX_7SUBTYPE": (40, "Maximum record 7 subtype"),
"SPSS_MAX_ENCODING": (64, "Maximum encoding text")}
supportedDates = { # uses ISO dates wherever applicable.
"DATE": "%Y-%m-%d",
"JDATE": "%Y-%m-%d",
"EDATE": "%Y-%m-%d",
"SDATE": "%Y-%m-%d",
"DATETIME": "%Y-%m-%d %H:%M:%S",
"WKDAY": "%A %H:%M:%S",
"ADATE": "%Y-%m-%d",
"WKDAY": "%A",
"MONTH": "%B",
"MOYR": "%B %Y",
"WKYR": "%W WK %Y"}
userMissingValues = {
"SPSS_NO_MISSVAL": 0,
"SPSS_ONE_MISSVAL": 1,
"SPSS_TWO_MISSVAL": 2,
"SPSS_THREE_MISSVAL": 3,
"SPSS_MISS_RANGE": -2,
"SPSS_MISS_RANGEANDVAL": -3}
###############################################################################
class SPSSIOError(Exception):
"""
Error class for the IBM SPSS Statistics Input Output Module
"""
def __init__(self, msg=u"Unknown", retcode=None):
if isinstance(retcode, int):
code = retcodes.get(retcode, retcode)
elif isinstance(retcode, list):
code = ", ".join([retcodes.get(retcodex) for retcodex in retcode])
else:
code = "unknown code %r" % retcode
msg = "%s [retcode: %r]" % (msg, code)
Exception.__init__(self, msg)
###############################################################################
def wide2utf8(fn):
"""Take a unicode file name string and encode it to a multibyte string
that Windows can use to represent file names (CP65001, UTF-8)
http://msdn.microsoft.com/en-us/library/windows/desktop/dd374130"""
from ctypes import wintypes
_CP_UTF8 = 65001
_CP_ACP = 0 # ANSI
_LPBOOL = POINTER(c_long)
_wideCharToMultiByte = windll.kernel32.WideCharToMultiByte
_wideCharToMultiByte.restype = c_int
_wideCharToMultiByte.argtypes = [wintypes.UINT, wintypes.DWORD,
wintypes.LPCWSTR, c_int, wintypes.LPSTR, c_int, wintypes.LPCSTR, _LPBOOL]
codePage = _CP_UTF8
dwFlags = 0
lpWideCharStr = fn
cchWideChar = len(fn)
lpMultiByteStr = None
cbMultiByte = 0 # zero requests size
lpDefaultChar = None
lpUsedDefaultChar = None
# get size
mbcssize = _wideCharToMultiByte(
codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
cbMultiByte, lpDefaultChar, lpUsedDefaultChar)
if mbcssize <= 0:
raise WinError(mbcssize)
lpMultiByteStr = create_string_buffer(mbcssize)
# convert
retcode = _wideCharToMultiByte(
codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
mbcssize, lpDefaultChar, lpUsedDefaultChar)
if retcode <= 0:
raise WinError(retcode)
return lpMultiByteStr.value
###############################################################################
class Generic(object):
"""
Class for methods and data used in reading as well as writing
IBM SPSS Statistics data files
"""
def __init__(self, savFileName, ioUtf8=False, ioLocale=None):
"""Constructor. Note that interface locale and encoding can only
be set once"""
locale.setlocale(locale.LC_ALL, "")
self.savFileName = savFileName
self.libc = cdll.LoadLibrary(ctypes.util.find_library("c"))
self.spssio = self.loadLibrary()
#self.myStruct = self.getStruct(self.varTypes, self.varNames)
#self.caseBuffer = self.getCaseBuffer()
self.wholeCaseIn = self.spssio.spssWholeCaseIn
#self.unpack_from = self.myStruct.unpack_from
self.wholeCaseOut = self.spssio.spssWholeCaseOut
#self.pack_into = self.myStruct.pack_into
self.encoding_and_locale_set = False
if not self.encoding_and_locale_set:
self.encoding_and_locale_set = True
self.ioLocale = ioLocale
self.ioUtf8 = ioUtf8
def _encodeFileName(self, fn):
"""Helper function to encode unicode file names into bytestring file
names encoded in the file system's encoding. Needed for C functions
that have a c_char_p filename argument.
http://effbot.org/pyref/sys.getfilesystemencoding.htm
http://docs.python.org/2/howto/unicode.html under 'unicode filenames'"""
if not isinstance(fn, unicode):
return fn
elif sys.platform.startswith("win"):
return wide2utf8(fn)
else:
encoding = sys.getfilesystemencoding()
encoding = "utf-8" if not encoding else encoding # actually, ascii
try:
return fn.encode(encoding)
except UnicodeEncodeError, e:
msg = ("File system encoding %r can not be used to " +
"encode file name %r [%s]")
raise ValueError(msg % (encoding, fn, e))
def loadLibrary(self):
"""This function loads and returns the SPSSIO libraries,
depending on the platform."""
path = os.path.abspath(".")
is_32bit = platform.architecture()[0] == "32bit"
pf = sys.platform.lower()
load = WinDLL if pf.startswith("win") else CDLL
if pf.startswith("win"):
os.environ["PATH"] += ";" + path
spssio = load("spssio32") if is_32bit else load("spssio64")
elif pf.startswith("lin"): # most linux flavours, incl zLinux
spssio = load("libspssdio.so.1")
elif pf.startswith("darwin") or pf.startswith("mac"):
spssio = load("libspssdio.dylib")
elif pf.startswith("aix") and not is_32bit:
spssio = load("libspssdio.so.1")
elif pf.startswith("hp-ux"):
spssio = load("libspssdio.sl.1")
elif pf.startswith("sunos") and not is_32bit:
spssio = load("libspssdio.so.1")
else:
msg = "Your platform (%r) is not supported" % pf
raise NotImplementedError(msg)
return spssio
def errcheck(self, res, func, args):
"""This function checks for errors during the execution of
function <func>"""
if not res:
msg = "Error performing %r operation on file %r."
raise IOError(msg % (func.__name__, self.savFileName))
return res
def openSavFile(self, savFileName, mode="rb", refSavFileName=None):
"""This function opens IBM SPSS Statistics data files in mode <mode>
and returns a handle that should be used for subsequent operations on
the file. If <savFileName> is opened in mode "cp", meta data
information aka the spss dictionary is copied from <refSavFileName>"""
savFileName = os.path.abspath(savFileName) # fdopen wants full name
try:
fdopen = self.libc._fdopen # Windows
except AttributeError:
fdopen = self.libc.fdopen # Linux and others
fdopen.argtypes = [c_int, c_char_p]
fdopen.restype = c_void_p
fdopen.errcheck = self.errcheck
with open(savFileName, mode) as f:
self.fd = fdopen(f.fileno(), mode)
if mode == "rb":
spssOpen = self.spssio.spssOpenRead
elif mode == "wb":
spssOpen = self.spssio.spssOpenWrite
elif mode == "cp":
spssOpen = self.spssio.spssOpenWriteCopy
elif mode == "ab":
spssOpen = self.spssio.spssOpenAppend
savFileName = self._encodeFileName(savFileName)
refSavFileName = self._encodeFileName(refSavFileName)
sav = c_char_p(savFileName)
fh = c_int(self.fd)
if mode == "cp":
retcode = spssOpen(sav, c_char_p(refSavFileName), pointer(fh))
else:
retcode = spssOpen(sav, pointer(fh))
if retcode > 0:
msg = "Error opening file %r in mode %r"
raise SPSSIOError(msg % (savFileName, mode), retcode)
return fh.value
def closeSavFile(self, fh, mode="rb"):
"""This function closes the sav file associated with <fh> that was open
in mode <mode>."""
if mode == "rb":
spssClose = self.spssio.spssCloseRead
elif mode == "wb":
spssClose = self.spssio.spssCloseWrite
elif mode == "ab":
spssClose = self.spssio.spssCloseAppend
retcode = spssClose(c_int(fh))
if retcode > 0:
raise SPSSIOError("Error closing file in mode %r" % mode, retcode)
def closeFile(self):
"""Close file descriptor"""
try:
if self.fd: # return value is not null
self.libc.fclose.errcheck = self.errcheck
self.libc.fclose(c_void_p(self.fd)) # error in Linux?
else:
try:
# Windows
self.libc._close.errcheck = self.errcheck
self.libc._close(c_void_p(self.fd))
except AttributeError:
# Linux
self.libc.close.errcheck = self.errcheck
self.libc.close(c_void_p(self.fd))
except EnvironmentError, e:
print e
@property
def releaseInfo(self):
"""This function reports release- and machine-specific information
about the open file."""
relInfo = ["release number", "release subnumber", "fixpack number",
"machine code", "floating-point representation code",
"compression scheme code", "big/little-endian code",
"character representation code"]
relInfoArr = (c_int * len(relInfo))()
retcode = self.spssio.spssGetReleaseInfo(c_int(self.fh), relInfoArr)
if retcode > 0:
raise SPSSIOError("Error getting ReleaseInfo", retcode)
info = dict([(item, relInfoArr[i]) for i, item in enumerate(relInfo)])
return info
@property
def spssVersion(self):
"""Return the SPSS version as a three-tuple indicating major,
minor, and fixpack version as ints. NB: in the transition from
SPSS to IBM, a new four-digit versioning nomenclature is used.
This function uses the old three-digit nomenclature. Therefore,
no patch version information is available."""
info = self.releaseInfo
major = info["release number"]
minor = info["release subnumber"]
fixpack = info["fixpack number"]
return major, minor, fixpack
@property
def systemString(self):
"""This function returns the name of the system under which the file
was created aa a string."""
sysName = create_string_buffer(42)
func = self.spssio.spssGetSystemString
retcode = func(c_int(self.fh), byref(sysName))
if retcode > 0:
raise SPSSIOError("Error getting SystemString", retcode)
return sysName.value
def getStruct(self, varTypes, varNames, mode="rb"):
"""This function returns a compiled struct object. The required
struct format string for the conversion between C and Python
is created on the basis of varType and byte order.
--varTypes: SPSS data files have either 8-byte doubles/floats or n-byte
chars[]/ strings, where n is always 8 bytes or a multiple thereof.
--byte order: files are written in the byte order of the host system
(mode="wb") and read/appended using the byte order information
contained in the SPSS data file (mode is "ab" or "rb" or "cp")"""
if mode in ("ab", "rb", "cp"): # derive endianness from file
endianness = self.releaseInfo["big/little-endian code"]
endianness = ">" if endianness > 0 else "<"
elif mode == "wb": # derive endianness from host
if sys.byteorder == "little":
endianness = "<"
elif sys.byteorder == "big":
endianness = ">"
else:
endianness = "@"
structFmt = [endianness]
ceil = math.ceil
for varName in varNames:
varType = varTypes[varName]
if varType == 0:
structFmt.append("d")
else:
fmt = str(int(ceil(int(varType) / 8.0) * 8))
structFmt.append(fmt + "s")
return struct.Struct("".join(structFmt))
def getCaseBuffer(self):
"""This function returns a buffer and a pointer to that buffer. A whole
case will be read into this buffer."""
caseSize = c_long()
retcode = self.spssio.spssGetCaseSize(c_int(self.fh), byref(caseSize))
caseBuffer = create_string_buffer(caseSize.value)
if retcode > 0:
raise SPSSIOError("Problem getting case buffer", retcode)
return caseBuffer
@property
def sysmis(self):
"""This function returns the IBM SPSS Statistics system-missing
value ($SYSMIS) for the host system (also called 'NA' in other
systems)."""
try:
sysmis = -1 * sys.float_info[0] # Python 2.6 and higher.
except AttributeError:
self.spssio.spssSysmisVal.restype = c_float
sysmis = self.spssio.spssSysmisVal()
return sysmis
@property
def missingValuesLowHigh(self):
"""This function returns the 'lowest' and 'highest' values used for
numeric missing value ranges on the host system. It may be called at
any time."""
lowest, highest = c_double(), c_double()
func = self.spssio.spssLowHighVal
retcode = func(byref(lowest), byref(highest))
return lowest.value, highest.value
@property
def ioLocale(self):
"""This function gets/sets the I/O Module's locale.
This corresponds with the SPSS command SET LOCALE. The I/O Module's
locale is separate from that of the client application. The
<localeName> parameter and the return value are identical to those
for the C run-time function setlocale. The exact locale name
specification depends on the OS of the host sytem, but has the
following form:
<lang>_<territory>.<codeset>[@<modifiers>]
The 'codeset' and 'modifier' components are optional and in Windows,
aliases (e.g. 'english') may be used. When the I/O Module is first
loaded, its locale is set to the system default. See also:
--https://wiki.archlinux.org/index.php/Locale
--http://msdn.microsoft.com/en-us/library/39cwe7zf(v=vs.80).aspx"""
if hasattr(self, "setLocale"):
return self.setLocale
else:
currLocale = ".".join(locale.getlocale())
print "NOTE. Locale not set; getting current locale: ", currLocale
return currLocale
@ioLocale.setter
def ioLocale(self, localeName=""):
if not localeName:
localeName = ".".join(locale.getlocale())
func = self.spssio.spssSetLocale
func.restype = c_char_p
self.setLocale = func(c_int(locale.LC_ALL), c_char_p(localeName))
if self.setLocale is None:
raise ValueError("Invalid ioLocale: %r" % localeName)
return self.setLocale
@property
def fileCodePage(self):
"""This function provides the Windows code page number of the encoding
applicable to a file."""
nCodePage = c_int()
func = self.spssio.spssGetFileCodePage
retcode = func(c_int(self.fh), byref(nCodePage))
return nCodePage.value
def isCompatibleEncoding(self):
"""This function determines whether the file and interface encoding
are compatible."""
try:
# Windows, note typo 'Endoding'!
func = self.spssio.spssIsCompatibleEndoding
except AttributeError:
func = self.spssio.spssIsCompatibleEncoding
func.restype = c_bool
isCompatible = c_int()
retcode = func(c_int(self.fh), byref(isCompatible))
if retcode > 0:
msg = "Error testing encoding compatibility: %r"
raise SPSSIOError(msg % isCompatible.value, retcode)
if not isCompatible.value and not self.ioUtf8:
msg = ("NOTE. SPSS Statistics data file %r is written in a " +
"character encoding (%s) incompatible with the current " +
"ioLocale setting. It may not be readable. Consider " +
"changing ioLocale or setting ioUtf8=True.")
print msg % (self.savFileName, self.fileEncoding)
return bool(isCompatible.value)
@property
def ioUtf8(self):
"""This function returns/sets the current interface encoding.
ioUtf8 = False --> CODEPAGE mode,
ioUtf8 = True --> UTF-8 mode, aka. Unicode mode
This corresponds with the SPSS command SHOW UNICODE (getter)
and SET UNICODE=ON/OFF (setter)."""
if hasattr(self, "ioUtf8_"):
return self.ioUtf8_
self.ioUtf8_ = self.spssio.spssGetInterfaceEncoding()
return bool(self.ioUtf8_)
@ioUtf8.setter
def ioUtf8(self, ioUtf8):
try:
retcode = self.spssio.spssSetInterfaceEncoding(c_int(int(ioUtf8)))
if retcode > 0 and not self.encoding_and_locale_set:
# not self.encoding_and_locale_set --> nested context managers
raise SPSSIOError("Error setting IO interface", retcode)
except TypeError:
msg = "Invalid interface encoding: %r (must be bool)"
raise SPSSIOError(msg % ioUtf8)
@property
def fileEncoding(self):
"""This function obtains the encoding applicable to a file.
The encoding is returned as an IANA encoding name, such as
ISO-8859-1, which is then converted to the corresponding Python
codec name. If the file contains no file encoding, the locale's
preferred encoding is returned"""
preferredEncoding = locale.getpreferredencoding()
try:
pszEncoding = create_string_buffer(20) # is 20 enough??
func = self.spssio.spssGetFileEncoding
retcode = func(c_int(self.fh), byref(pszEncoding))
if retcode > 0:
raise SPSSIOError("Error getting file encoding", retcode)
iana_codes = encodings.aliases.aliases
rawEncoding = pszEncoding.value.lower()
if rawEncoding.replace("-", "") in iana_codes:
iana_code = rawEncoding.replace("-", "")
else:
iana_code = rawEncoding.replace("-", "_")
fileEncoding = iana_codes[iana_code]
return fileEncoding
except AttributeError:
print ("NOTE. Function 'getFileEncoding' not found. You are " +
"using a .dll from SPSS < v16.")
return preferredEncoding
except KeyError:
print ("NOTE. IANA coding lookup error. Code %r " % iana_code +
"does not map to any Python codec.")
return preferredEncoding
@property
def record(self):
"""Get/Set a whole record from/to a pre-allocated buffer"""
retcode = self.wholeCaseIn(c_int(self.fh),
byref(self.caseBuffer))
if retcode > 0:
raise SPSSIOError("Problem reading row", retcode)
record = list(self.unpack_from(self.caseBuffer))
return record
@record.setter
def record(self, record):
try:
self.pack_into(self.caseBuffer, 0, *record)
except struct.error, e:
msg = "Use ioUtf8=True to write unicode strings [%s]" % e
raise TypeError(msg)
retcode = self.wholeCaseOut(c_int(self.fh),
c_char_p(self.caseBuffer.raw))
if retcode > 0:
raise SPSSIOError("Problem writing row:\n" + record, retcode)
def printPctProgress(self, nominator, denominator):
"""This function prints the % progress when reading and writing
files"""
if nominator and nominator % 10**4 == 0:
pctProgress = (float(nominator) / denominator) * 100
print "%2.1f%%... " % pctProgress,
###############################################################################
class Header(Generic):
"""
This class contains methods responsible for getting and setting meta data
that is embedded in the IBM SPSS Statistics data file. In SPSS speak, this
header information is known as the SPSS Data Dictionary (which has diddly
squat to do with a Python dictionary!).
"""
def __init__(self, savFileName, mode, refSavFileName, ioUtf8, ioLocale=None):
"""Constructor"""
super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
self.freeMemoryItems = {}
self.spssio = self.loadLibrary()
self.fh = super(Header, self).openSavFile(savFileName, mode,
refSavFileName)
self.varNames, self.varTypes = self.varNamesTypes
self.vNames = dict(zip(self.varNames, self.encode(self.varNames)))
def openSavFile(self):
"""This function returns the file handle that was opened in the
super class"""
return self.fh
def decode(func):
"""Decorator to Utf-8 decode all str items contained in a dictionary
If ioUtf8=True, the dictionary's keys and values are decoded, but only
values that are strs, lists, or dicts. For example:
{'v1': {'y': 'yy', 'z': 666}}-->{u'v1': {u'y': u'yy', u'z': 666}}"""
uS = lambda x: x.decode("utf-8") if isinstance(x, str) else x
uL = lambda x: map(uS, x) if isinstance(x, list) else x
@functools.wraps(func)
def wrapper(arg):
result = func(arg)
if not arg.ioUtf8:
return result # unchanged
if isinstance(result, str):
return uS(result)
utf8ifiedDict = {}
for k, v in result.iteritems():
k, v = uS(k), uS(v)
if isinstance(v, list):
v = map(uS, v)
elif isinstance(v, dict):
v = dict([(uS(x), uL(uS(y))) for x, y in v.items()])
utf8ifiedDict[k] = v
return utf8ifiedDict
return wrapper
def encode(self, item):
"""Counter part of decode helper function, does the opposite of that
function (but is not a decorator)"""
if not self.ioUtf8:
return item # unchanged
utf8dify = lambda x: x.encode("utf-8") if isinstance(x, unicode) else x
if isinstance(item, list):
return map(utf8dify, item)
elif isinstance(item, dict):
return dict([(utf8dify(x), utf8dify(y)) for x, y in item.items()])
return utf8dify(item)
def freeMemory(self, items):
"""Clean up: free memory claimed by e.g. getValueLabels and
getVarNamesTypes"""
# Previously crashed under other systems than WinXP (eg. Linux & Win7)
# This function actually has made my receding hairline worse ;-)
retcodex = []
isWindows = sys.platform.startswith("win")
pfVersion = platform.version().split(".")[0]
pfVersion = int(pfVersion) if pfVersion.isdigit() else 5 # 5 = WinXP
for funcName, args in sorted(items.iteritems()):
func = getattr(self.spssio, funcName.split()[0])
if isWindows and pfVersion <= 5:
retcode = func(*args)
retcodex.append(retcode)
else: # Linux Ubuntu and Windows 7, probably more
try:
# free arrays element-by-element
array1, array2, num = args
for element1, element2 in zip(array1, array2):
retcode = func(element1, element2, num)
retcodex.append(retcode)
except ValueError:
retcode = func(*args)
retcodex.append(retcode)
okay_or_not = "NOT okay!!!" if retcode > 0 else "okay"
print "...freeing %s --> %s" % (funcName[8:], okay_or_not)
retcode = max(retcodex)
if retcode > 0:
raise SPSSIOError("Error freeing memory", retcode)
@property
def numberofCases(self):
"""This function reports the number of cases present in a data file.
Prehistoric files (< SPSS v6.0) don't contain nCases info, therefore
a guesstimate of the number of cases is given for those files"""
nCases = c_long()
func = self.spssio.spssGetNumberofCases
retcode = func(c_int(self.fh), byref(nCases))
if nCases.value == -1:
func = self.spssio.spssGetEstimatedNofCases
retcode = func(c_int(self.fh), byref(nCases))
if retcode > 0:
raise SPSSIOError("Error getting number of cases", retcode)
return nCases.value
@property
def numberofVariables(self):
"""This function returns the number of variables (columns) in the
spss dataset"""
numVars = c_int()
func = self.spssio.spssGetNumberofVariables
retcode = func(c_int(self.fh), byref(numVars))
if retcode > 0:
raise SPSSIOError("Error getting number of variables", retcode)
return numVars.value
@property
def varNamesTypes(self):
"""Get/Set variable names and types.
--Variable names is a list of the form ['var1', var2', 'etc']
--Variable types is a dictionary of the form {varName: varType}
The variable type code is an integer in the range 0-32767, 0
indicating a numeric variable (F8.2) and a positive value
indicating a string variable of that size (in bytes)."""
if hasattr(self, "varNames"):
return self.varNames, self.varTypes
numVars = self.numberofVariables
numVars_ = c_int(numVars)
varNamesArr = (POINTER(c_char_p) * numVars)()
varTypesArr = (POINTER(c_int) * numVars)()
func = self.spssio.spssGetVarNames
retcode = func(c_int(self.fh), byref(numVars_),
byref(varNamesArr), byref(varTypesArr))
if retcode > 0:
raise SPSSIOError("Error getting variable names & types", retcode)
varNames = [varNamesArr[0][i] for i in xrange(numVars)]
varTypes = [varTypesArr[0][i] for i in xrange(numVars)]
if self.ioUtf8:
varNames = [varName.decode("utf-8") for varName in varNames]
funcStr = "spssFreeVarNames"
self.freeMemoryItems[funcStr] = (varNamesArr, varTypesArr, numVars)
return varNames, dict(zip(varNames, varTypes))
@varNamesTypes.setter
def varNamesTypes(self, varNamesVarTypes):
badLengthMsg = "Empty or longer than %s chars" % \
(MAXLENGTHS['SPSS_MAX_VARNAME'][0])
varNames, varTypes = varNamesVarTypes
varNameRetcodes = {
0: ('SPSS_NAME_OK', 'Valid standard name'),
1: ('SPSS_NAME_SCRATCH', 'Valid scratch var name'),
2: ('SPSS_NAME_SYSTEM' 'Valid system var name'),
3: ('SPSS_NAME_BADLTH', badLengthMsg),
4: ('SPSS_NAME_BADCHAR', 'Invalid character or embedded blank'),
5: ('SPSS_NAME_RESERVED', 'Name is a reserved word'),
6: ('SPSS_NAME_BADFIRST', 'Invalid initial char (otherwise OK)')}
validate = self.spssio.spssValidateVarname
func = self.spssio.spssSetVarName
for varName in self.varNames:
varLength = self.varTypes[varName]
retcode = validate(c_char_p(varName))
if retcode > 0:
msg = "%r is an invalid variable name [%r]" % \
(varName, ": ".join(varNameRetcodes.get(retcode)))
raise SPSSIOError(msg, retcode)
retcode = func(c_int(self.fh), c_char_p(varName), c_int(varLength))
if retcode > 0:
msg = "Problem setting variable name %r" % varName
raise SPSSIOError(msg, retcode)
@property
@decode
def valueLabels(self):
"""Get/Set VALUE LABELS.
Takes a dictionary of the form {varName: {value: valueLabel}:
--{'numGender': {1: 'female', {2: 'male'}}
--{'strGender': {'f': 'female', 'm': 'male'}}"""
MAX_ARRAY_SIZE = 1000 # assume never more than 1000 labels
numLabels, valueLabels = c_int(), {}
for varName in self.varNames:
labelsArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
if self.varTypes[varName] == 0:
valuesArr = (POINTER(c_double) * MAX_ARRAY_SIZE)()
func = self.spssio.spssGetVarNValueLabels
funcStr = "spssFreeVarNValueLabels"
items = (valuesArr, labelsArr, numLabels)
self.freeMemoryItems[funcStr] = items
else:
valuesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
func = self.spssio.spssGetVarCValueLabels
funcStr = "spssFreeVarCValueLabels"
items = (valuesArr, labelsArr, numLabels)
self.freeMemoryItems[funcStr] = items
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName), byref(valuesArr),
byref(labelsArr), byref(numLabels))
if retcode > 0:
msg = "Error getting value labels of variable %r"
raise SPSSIOError(msg % varName, retcode)
values = [valuesArr[0][i] for i in xrange(numLabels.value)]
labels = [labelsArr[0][i] for i in xrange(numLabels.value)]
valueLabelsX = [(val, lbl) for val, lbl in zip(values, labels)]
valueLabels[varName] = dict(valueLabelsX)
return valueLabels
@valueLabels.setter
def valueLabels(self, valueLabels):
if not valueLabels:
return
valLabN = self.spssio.spssSetVarNValueLabel
valLabC = self.spssio.spssSetVarCValueLabel
valueLabels = self.encode(valueLabels)
for varName, valueLabelsX in valueLabels.iteritems():
valueLabelsX = self.encode(valueLabelsX)
for value, label in valueLabelsX.iteritems():
if self.varTypes[varName] == 0:
retcode = valLabN(c_int(self.fh), c_char_p(varName),
c_double(value), c_char_p(label))
else:
retcode = valLabC(c_int(self.fh), c_char_p(varName),
c_char_p(value), c_char_p(label))
if retcode > 0:
msg = "Problem with setting value labels of variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def varLabels(self):
"""Get/set VARIABLE LABELS.
Returns/takes a dictionary of the form {varName: varLabel}.
For example: varLabels = {'salary': 'Salary (dollars)',
'educ': 'Educational level (years)'}"""
SPSS_VARLABEL_SHORT = 120 # fixed amount
funcS = self.spssio.spssGetVarLabel
funcL = self.spssio.spssGetVarLabelLong
varLabels = {}
for varName in self.varNames:
varLabel = create_string_buffer(SPSS_VARLABEL_SHORT + 1)
vName = self.vNames[varName]
retcode = funcS(c_int(self.fh), c_char_p(vName), byref(varLabel))
if varLabel.value and len(varLabel.value) > SPSS_VARLABEL_SHORT:
lenBuff = MAXLENGTHS['SPSS_MAX_VARLABEL']
varLabel = create_string_buffer(lenBuff)
retcode = funcL(c_int(self.fh), c_char_p(varName),
byref(varLabel), c_int(lenBuff), byref(c_int))
varLabels[varName] = varLabel.value
if retcode > 0:
msg = "Error getting variable label of variable %r" % varName
raise SPSSIOError(msg, retcode)
return varLabels
@varLabels.setter
def varLabels(self, varLabels):
if not varLabels:
return
func = self.spssio.spssSetVarLabel
varLabels = self.encode(varLabels)
for varName, varLabel in varLabels.iteritems():
retcode = func(c_int(self.fh), c_char_p(varName),
c_char_p(varLabel))
if retcode > 0:
msg = ("Problem with setting variable label %r of variable %r"
% (varLabel, varName))
raise SPSSIOError(msg, retcode)
@property
@decode
def formats(self):
"""Get the PRINT FORMATS, set PRINT and WRITE FORMATS.
Returns/takes a dictionary of the form {varName: <format_>.
For example: formats = {'salary': 'DOLLAR8', 'gender': 'A1',
'educ': 'F8.2'}"""
if hasattr(self, "varformats"):
return self.varformats
printFormat_, printDec_, printWid_ = c_int(), c_int(), c_int()
func = self.spssio.spssGetVarPrintFormat
formats = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(printFormat_), byref(printDec_),
byref(printWid_))
if retcode > 0:
msg = "Error getting print format for variable %r"
raise SPSSIOError(msg % vName, retcode)
printFormat = allFormats.get(printFormat_.value)[0]
printFormat = printFormat.split("_")[-1]
format_ = printFormat + str(printWid_.value)
if self.varTypes[varName] == 0:
format_ += ("." + str(printDec_.value))
if format_.endswith(".0"):
format_ = format_[:-2]
formats[varName] = format_
return formats
def _splitformats(self):
"""This function returns the 'bare' formats + variable widths,
e.g. format_ F5.3 is returned as 'F' and '5'"""
regex = re.compile("\w+(?P<varWid>\d+)[.]?\d?", re.I)
bareformats, varWids = {}, {}
for varName, format_ in self.formats.iteritems():
bareformats[varName] = re.sub(r"\d+.?", "", format_)
varWids[varName] = int(regex.search(format_).group("varWid"))
return bareformats, varWids
@formats.setter
def formats(self, formats):
if not formats:
return
reverseFormats = dict([(v[0][9:], k) for k, v in allFormats.items()])
validValues = sorted(reverseFormats.keys())
regex = "(?P<printFormat>A(HEX)?)(?P<printWid>\d+)"
isStringVar = re.compile(regex, re.IGNORECASE)
regex = "(?P<printFormat>[A-Z]+)(?P<printWid>\d+)\.?(?P<printDec>\d*)"
isAnyVar = re.compile(regex, re.IGNORECASE)
funcP = self.spssio.spssSetVarPrintFormat # print type
funcW = self.spssio.spssSetVarWriteFormat # write type
for varName, format_ in self.encode(formats).iteritems():
format_ = format_.upper()
gotString = isStringVar.match(format_)
gotAny = isAnyVar.match(format_)
msg = ("Unknown format_ %r for variable %r. " +
"Valid formats are: %s")
msg = msg % (", ".join(validValues), format_, varName)
if gotString:
printFormat = gotString.group("printFormat")
printFormat = reverseFormats.get(printFormat)
printDec = 0
printWid = int(gotString.group("printWid"))
elif gotAny:
printFormat = gotAny.group("printFormat")
printFormat = reverseFormats.get(printFormat)
printDec = gotAny.group("printDec")
printDec = int(printDec) if printDec else 0
printWid = int(gotAny.group("printWid"))
else:
raise ValueError(msg)
if printFormat is None:
raise ValueError(msg)
args = (c_int(self.fh), c_char_p(varName), c_int(printFormat),
c_int(printDec), c_int(printWid))
retcode1, retcode2 = funcP(*args), funcW(*args)
if retcodes.get(retcode1) == "SPSS_INVALID_PRFOR":
# invalid PRint FORmat
msg = "format_ for %r misspecified (%r)"
raise SPSSIOError(msg % (varName, format_), retcode1)
elif retcode1 > 0:
msg = "Error setting format_ %r for %r"
raise SPSSIOError(msg % (format_, varName), retcode1)
def _getMissingValue(self, varName):
"""This is a helper function for the missingValues getter
method. The function returns the missing values of variable <varName>
as a a dictionary. The dictionary keys and items depend on the
particular definition, which may be discrete values and/or ranges.
Range definitions are only possible for numerical variables."""
if self.varTypes[varName] == 0:
func = self.spssio.spssGetVarNMissingValues
args = (c_double(), c_double(), c_double())
else:
func = self.spssio.spssGetVarCMissingValues
args = (create_string_buffer(9), create_string_buffer(9),
create_string_buffer(9))
missingFmt = c_int()
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(missingFmt), *map(byref, args))
if retcode > 0:
msg = "Error getting missing value for variable %r"
raise SPSSIOError(msg % varName, retcode)
v1, v2, v3 = [v.value for v in args]
userMiss = dict([(v, k) for k, v in userMissingValues.iteritems()])
missingFmt = userMiss[missingFmt.value]
if missingFmt == "SPSS_NO_MISSVAL":
return {}
elif missingFmt == "SPSS_ONE_MISSVAL":
return {"values": [v1]}
elif missingFmt == "SPSS_TWO_MISSVAL":
return {"values": [v1, v2]}
elif missingFmt == "SPSS_THREE_MISSVAL":
return {"values": [v1, v2, v3]}
elif missingFmt == "SPSS_MISS_RANGE":
return {"lower": v1, "upper": v2}
elif missingFmt == "SPSS_MISS_RANGEANDVAL":
return {"lower": v1, "upper": v2, "value": v3}
def _setMissingValue(self, varName, **kwargs):
"""This is a helper function for the missingValues setter
method. The function sets missing values for variable <varName>.
Valid keyword arguments are:
* to specify a RANGE: 'lower', 'upper', optionally with 'value'
* to specify DISCRETE VALUES: 'values', specified as a list no longer
than three items, or as None, or as a float/int/str
"""
if kwargs == {}:
return 0
fargs = ["lower", "upper", "value", "values"]
if set(kwargs.keys()).difference(set(fargs)):
raise ValueError("Allowed keywords are: %s" % ", ".join(fargs))
varName = self.encode(varName)
varType = self.varTypes[varName]
# range of missing values, e.g. MISSING VALUES aNumVar (-9 THRU -1).
if varType == 0:
placeholder = 0.0
if "lower" in kwargs and "upper" in kwargs and "value" in kwargs:
missingFmt = "SPSS_MISS_RANGEANDVAL"
args = kwargs["lower"], kwargs["upper"], kwargs["value"]
elif "lower" in kwargs and "upper" in kwargs:
missingFmt = "SPSS_MISS_RANGE"
args = kwargs["lower"], kwargs["upper"], placeholder
else:
placeholder, args = "0", None
# up to three discrete missing values
if "values" in kwargs:
values = self.encode(kwargs.values()[0])
if isinstance(values, (float, int, str)):
values = [values]
# check if missing values strings values are not too long
strMissLabels = [len(v) for v in values if isinstance(v, str)]
if strMissLabels and max(strMissLabels) > 9:
raise ValueError("Missing value label > 9 bytes")
nvalues = len(values) if values is not None else values
if values is None or values == {}:
missingFmt = "SPSS_NO_MISSVAL"
args = placeholder, placeholder, placeholder
elif nvalues == 1:
missingFmt = "SPSS_ONE_MISSVAL"
args = values + [placeholder, placeholder]
elif nvalues == 2:
missingFmt = "SPSS_TWO_MISSVAL"
args = values + [placeholder]
elif nvalues == 3:
missingFmt = "SPSS_THREE_MISSVAL"
args = values
else:
msg = "You can specify up to three individual missing values"
raise ValueError(msg)
# numerical vars
if varType == 0 and args:
func = self.spssio.spssSetVarNMissingValues
func.argtypes = [c_int, c_char_p, c_int,
c_double, c_double, c_double]
args = map(float, args)
# string vars
else:
if args is None:
raise ValueError("Illegal keyword for character variable")
func = self.spssio.spssSetVarCMissingValues
func.argtypes = [c_int, c_char_p, c_int,
c_char_p, c_char_p, c_char_p]
retcode = func(self.fh, varName, userMissingValues[missingFmt], *args)
if retcode > 0:
msg = "Problem setting missing value of variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def missingValues(self):
"""Get/Set MISSING VALUES.
User missing values are values that will not be included in
calculations by SPSS. For example, 'don't know' might be coded as a
user missing value (a value of 999 is typically used, so when vairable
'age' has values 5, 15, 999, the average age is 10). This is
different from 'system missing values', which are blank/null values.
Takes a dictionary of the following form:
{'someNumvar1': {'values': [999, -1, -2]}, # discrete values
'someNumvar2': {'lower': -9, 'upper':-1}, # range "-9 THRU -1"
'someNumvar3': {'lower': -9, 'upper':-1, 'value': 999},
'someStrvar1': {'values': ['foo', 'bar', 'baz']},
'someStrvar2': {'values': 'bletch'}} # shorthand """
missingValues = {}
for varName in self.varNames:
missingValues[varName] = self._getMissingValue(varName)
return missingValues
@missingValues.setter
def missingValues(self, missingValues):
if missingValues:
for varName, kwargs in missingValues.iteritems():
self._setMissingValue(varName, **kwargs)
# measurelevel, colwidth and alignment must all be set or not at all.
@property
@decode
def measureLevels(self):
"""Get/Set VARIABLE LEVEL (measurement level).
Returns/Takes a dictionary of the form {varName: varMeasureLevel}.
Valid measurement levels are: "unknown", "nominal", "ordinal", "scale",
"ratio", "flag", "typeless". This is used in Spss procedures such as
CTABLES."""
func = self.spssio.spssGetVarMeasureLevel
levels = {0: "unknown", 1: "nominal", 2: "ordinal", 3: "scale",
3: "ratio", 4: "flag", 5: "typeless"}
measureLevel = c_int()
varMeasureLevels = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(measureLevel))
varMeasureLevels[varName] = levels.get(measureLevel.value)
if retcode > 0:
msg = "Error getting variable measurement level: %r"
raise SPSSIOError(msg % varName, retcode)
return varMeasureLevels
@measureLevels.setter
def measureLevels(self, varMeasureLevels):
if not varMeasureLevels:
return
func = self.spssio.spssSetVarMeasureLevel
levels = {"unknown": 0, "nominal": 1, "ordinal": 2, "scale": 3,
"ratio": 3, "flag": 4, "typeless": 5}
for varName, level in self.encode(varMeasureLevels).iteritems():
if level.lower() not in levels:
msg = "Valid levels are %"
raise ValueError(msg % ", ".join(levels.keys()))
level = levels.get(level.lower())
retcode = func(c_int(self.fh), c_char_p(varName), c_int(level))
if retcode > 0:
msg = ("Error setting variable mesasurement level. " +
"Valid levels are: %s")
raise SPSSIOError(msg % ", ".join(levels.keys()), retcode)
@property
@decode
def columnWidths(self):
"""Get/Set VARIABLE WIDTH (display width).
Returns/Takes a dictionary of the form {varName: <int>}. A value of
zero is special and means that the IBM SPSS Statistics Data Editor
is to set an appropriate width using its own algorithm. If used,
variable alignment, measurement level and column width all needs to
be set."""
func = self.spssio.spssGetVarColumnWidth
varColumnWidth = c_int()
varColumnWidths = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(varColumnWidth))
if retcode > 0:
msg = ("Error getting column width: %r" % varName)
raise SPSSIOError(msg, retcode)
varColumnWidths[varName] = varColumnWidth.value
return varColumnWidths
@columnWidths.setter
def columnWidths(self, varColumnWidths):
if not varColumnWidths:
return
func = self.spssio.spssSetVarColumnWidth
for varName, varColumnWidth in varColumnWidths.iteritems():
retcode = func(c_int(self.fh), c_char_p(varName),
c_int(varColumnWidth))
if retcode > 0:
msg = "Error setting variable colunm width"
raise SPSSIOError(msg, retcode)
def _setColWidth10(self):
"""Set the variable display width of string values to at least 10
(it's annoying that SPSS displays e.g. a one-character variable in
very narrow columns). This also sets all measurement levels to
"unknown" and all variable alignments to "left". This function is
only called if column widths, measurement levels and variable
alignments are None."""
columnWidths = {}
for varName, varType in self.varTypes.iteritems():
# zero = appropriate width determined by spss
columnWidths[varName] = 10 if 0 < varType < 10 else 0
self.columnWidths = columnWidths
self.measureLevels = dict([(v, "unknown") for v in self.varNames])
self.alignments = dict([(v, "left") for v in self.varNames])
@property
@decode
def alignments(self):
"""Get/Set VARIABLE ALIGNMENT.
Returns/Takes a dictionary of the form {varName: alignment}
Valid alignment values are: left, right, center. If used, variable
alignment, measurement level and column width all need to be set.
"""
func = self.spssio.spssGetVarAlignment
alignments = {0: "left", 1: "right", 2: "center"}
alignment_ = c_int()
varAlignments = {}
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(alignment_))
alignment = alignments[alignment_.value]
varAlignments[varName] = alignment
if retcode > 0:
msg = ("Error getting variable alignment: %r" % varName)
raise SPSSIOError(msg, retcode)
return varAlignments
@alignments.setter
def alignments(self, varAlignments):
if not varAlignments:
return
func = self.spssio.spssSetVarAlignment
alignments = {"left": 0, "right": 1, "center": 2}
for varName, varAlignment in varAlignments.iteritems():
if varAlignment.lower() not in alignments:
msg = "Valid alignments are %"
raise ValueError(msg % ", ".join(alignments.keys()))
alignment = alignments.get(varAlignment.lower())
retcode = func(c_int(self.fh), c_char_p(varName), c_int(alignment))
if retcode > 0:
msg = "Error setting variable alignment for variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def varSets(self):
"""Get/Set VARIABLE SET information.
Returns/Takes a dictionary with SETNAME as keys and a list of SPSS
variables as values. For example: {'SALARY': ['salbegin',
'salary'], 'DEMOGR': ['gender', 'minority', 'educ']}"""
varSets = c_char_p()
func = self.spssio.spssGetVariableSets
retcode = func(c_int(self.fh), byref(varSets))
if retcode > 0:
msg = "Problem getting variable set information"
raise SPSSIOError(msg, retcode)
elif retcode == 0:
varSets_ = {}
for varSet in varSets.value.split("\n")[:-1]:
k, v = varSet.split("= ")
varSets_[k] = v.split()
self.freeMemoryItems["spssFreeVariableSets"] = (varSets,)
return varSets_
return {}
@varSets.setter
def varSets(self, varSets):
if not varSets:
return
varSets_ = []
for varName, varSet in varSets.iteritems():
varSets_.append("%s= %s" % (varName, " ".join(varSet)))
varSets_ = c_char_p("\n".join(varSets_))
retcode = self.spssio.spssSetVariableSets(c_int(self.fh), varSets_)
if retcode > 0:
msg = "Problem setting variable set information"
raise SPSSIOError(msg, retcode)
@property
@decode
def varRoles(self):
"""Get/Set VARIABLE ROLES.
Returns/Takes a dictionary of the form {varName: varRole}, where
varRoles may be any of the following: 'both', 'frequency', 'input',
'none', 'partition', 'record ID', 'split', 'target'"""
func = self.spssio.spssGetVarRole
roles = {0: "input", 1: "target", 2: "both", 3: "none", 4: "partition",
5: "split", 6: "frequency", 7: "record ID"}
varRoles = {}
varRole_ = c_int()
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName), byref(varRole_))
varRole = roles.get(varRole_.value)
varRoles[varName] = varRole
if retcode > 0:
msg = "Problem getting variable role for variable %r"
raise SPSSIOError(msg % varName, retcode)
return varRoles
@varRoles.setter
def varRoles(self, varRoles):
if not varRoles:
return
roles = {"input": 0, "target": 1, "both": 2, "none": 3, "partition": 4,
"split": 5, "frequency": 6, "record ID": 7}
func = self.spssio.spssSetVarRole
for varName, varRole in varRoles.iteritems():
varRole = roles.get(varRole)
retcode = func(c_int(self.fh), c_char_p(varName), c_int(varRole))
if retcode > 0:
msg = "Problem setting variable role %r for variable %r"
raise SPSSIOError(msg % (varRole, varName), retcode)
@property
@decode
def varAttributes(self):
"""Get/Set VARIABLE ATTRIBUTES.
Returns/Takes dictionary of the form:
{'var1': {'attr name x': 'attr value x','attr name y': 'attr value y'},
'var2': {'attr name a': 'attr value a','attr name b': 'attr value b'}}
"""
numVars = len(self.varNames)
attrNamesArr = (POINTER(c_char_p) * numVars)()
attrValuesArr = (POINTER(c_char_p) * numVars)()
nAttr, attributes = c_int(), {}
func = self.spssio.spssGetVarAttributes
for varName in self.varNames:
vName = self.vNames[varName]
retcode = func(c_int(self.fh), c_char_p(vName),
byref(attrNamesArr), byref(attrValuesArr),
byref(nAttr))
if retcode > 0:
msg = "Problem getting variable attributes for variable %r"
raise SPSSIOError(msg % varName, retcode)
k, v, n = attrNamesArr[0], attrValuesArr[0], nAttr.value
attributes[varName] = dict([(k[i], v[i]) for i in xrange(n)])
funcStr = "spssFreeAttributes varAttributes"
self.freeMemoryItems[funcStr] = (attrNamesArr, attrValuesArr, nAttr)
return attributes
@varAttributes.setter
def varAttributes(self, varAttributes):
if not varAttributes:
return
func = self.spssio.spssSetVarAttributes
for varName in self.varNames:
attributes = varAttributes.get(varName)
if not attributes:
continue
nAttr = len(attributes)
attrNames = (c_char_p * nAttr)(*attributes.keys())
attrValues = (c_char_p * nAttr)(*attributes.values())
retcode = func(c_int(self.fh), c_char_p(varName),
pointer(attrNames), pointer(attrValues),
c_int(nAttr))
if retcode > 0:
msg = "Problem setting variable attributes for variable %r"
raise SPSSIOError(msg % varName, retcode)
@property
@decode
def fileAttributes(self):
"""Get/Set DATAFILE ATTRIBUTES.
Returns/Takes a dictionary of the form:
{'attrName[1]': 'attrValue1', 'revision[1]': '2010-10-09',
'revision[2]': '2010-10-22', 'revision[3]': '2010-11-19'}
"""
MAX_ARRAY_SIZE = 100
nAttr = c_int()
attrNamesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
attrValuesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
func = self.spssio.spssGetFileAttributes
retcode = func(c_int(self.fh), byref(attrNamesArr),
byref(attrValuesArr), byref(nAttr))
if retcode > 0:
raise SPSSIOError("Problem getting file attributes", retcode)
k, v = attrNamesArr[0], attrValuesArr[0]
attributes = dict([(k[i], v[i]) for i in xrange(nAttr.value)])
funcStr = "spssFreeAttributes fileAttributes"
self.freeMemoryItems[funcStr] = (attrNamesArr, attrValuesArr, nAttr)
return attributes
@fileAttributes.setter
def fileAttributes(self, fileAttributes):
if not fileAttributes:
return
attributes, valueLens = {}, []
for name, values in fileAttributes.iteritems():
valueLens.append(len(values))
for value in values:
attributes[name] = value
#nAttr = len(fileAttributes)
nAttr = max(valueLens) # n elements per vector. But this may vary??
attrNames = (c_char_p * nAttr)(*attributes.keys())
attrValues = (c_char_p * nAttr)(*attributes.values())
func = self.spssio.spssSetFileAttributes
retcode = func(c_int(self.fh), pointer(attrNames),
pointer(attrValues), c_int(nAttr))
if retcode > 0:
raise SPSSIOError("Problem setting file attributes", retcode)
def _getMultRespDef(self, mrDef):
"""Get 'normal' multiple response defintions.
This is a helper function for the multRespDefs getter function.
A multiple response definition <mrDef> in the string format returned
by the IO module is converted into a multiple response definition of
the form multRespSet = {<setName>: {"setType": <setType>, "label":
<lbl>, "varNames": <list_of_varNames>}}. SetType may be either 'D'
(multiple dichotomy sets) or 'C' (multiple category sets). If setType
is 'D', the multiple response definition also includes '"countedValue":
countedValue'"""
regex = "\$(?P<setName>\w+)=(?P<setType>[CD])\n?"
regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
m = re.search(regex + ".*", mrDef, re.I | re.U)
if not m:
return {}
setType = m.group("setType")
if setType == "C": # multiple category sets
regex += " (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?"
regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
matches = re.findall(regex, mrDef, re.I)
setName, setType, lblLen, lblVarNames = matches[0]
else: # multiple dichotomy sets
regex += ("(?P<valueLen>\d+) (?P<countedValue>\w+)" +
" (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?")
regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
matches = re.findall(regex, mrDef, re.I | re.U)
setName, setType, valueLen = matches[0][:3]
countedValue, lblLen, lblVarNames = matches[0][3:]
lbl = lblVarNames[:int(lblLen)]
varNames = lblVarNames[int(lblLen):].split()
multRespSet = {setName: {"setType": setType, "label": lbl,
"varNames": varNames}}
if setType == "D":
multRespSet[setName]["countedValue"] = countedValue
return multRespSet
def _setMultRespDefs(self, multRespDefs):
"""Set 'normal' multiple response defintions.
This is a helper function for the multRespDefs setter function.
It translates the multiple response definition, specified as a
dictionary, into a string that the IO module can use"""
mrespDefs = []
for setName, rest in multRespDefs.iteritems():
if rest["setType"] not in ("C", "D"):
return
rest["setName"] = setName
mrespDef = "$%(setName)s=%(setType)s" % rest
lblLen = len(mrespDef["label"])
rest["lblLen"] = lblLen
rest["varNames"] = " ".join(rest["varNames"])
tail = "%(varNames)s" if lblLen == 0 else "%(label)s %(varNames)s"
if mrespDef["setType"] == "C": # multiple category sets
template = " %%(lblLen)s %s " % tail
else: # multiple dichotomy sets
template = " %%(countedValue)s %%(lblLen)s %s " % tail
mrespDef += template % rest
mrespDefs.append(mrespDef)
mrespDefs = "\n".join(mrespDefs)
return mrespDefs
def _getMultRespDefsEx(self, mrDef):
"""Get 'extended' multiple response defintions.
This is a helper function for the multRespDefs getter function."""
regex = ("\$(?P<setName>\w+)=(?P<setType>E) (?P<flag1>1)" +
"(?P<flag2>1)? (?P<valueLen>[0-9]+) (?P<countedValue>\w+) " +
"(?P<lblLen>[0-9]+) (?P<lblVarNames>[\w ]+)")
regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
matches = re.findall(regex, mrDef, re.I | re.U)
setName, setType, flag1, flag2 = matches[0][:4]
valueLen, countedValue, lblLen, lblVarNames = matches[0][4:]
length = int(lblLen)
label, varNames = lblVarNames[:length], lblVarNames[length:].split()
return {setName: {"setType": setType, "firstVarIsLabel": bool(flag2),
"label": label, "countedValue": countedValue,
"varNames": varNames}}
def _setMultRespDefsEx(self, multRespDefs):
"""Set 'extended' multiple response defintions.
This is a helper function for the multRespDefs setter function."""
mrDefs = []
for setName, rest in multRespDefs.iteritems():
if rest["setType"] != "E":
return {}
rest["setName"] = setName
v = int(rest["firstVarIsLabel"])
rest["firstVarIsLabel"] = v if v == 1 else ""
rest["valueLen"] = len(rest["countedValue"])
rest["lblLen"] = len(rest["label"])
rest["label"] = rest["label"]
rest["varNames"] = " ".join(rest["varNames"])
mrDef = "$%(setName)s=%(setType)s 1%(firstVarIsLabel)s "
mrDef += "%(valueLen)s %(countedValue)s %(lblLen)s %(label)s "
mrDef += "%(varNames)s"
mrDefs.append((mrDef % rest).replace(" ", " "))
return "\n".join(mrDefs)
@property
@decode
def multRespDefs(self):
"""Get/Set MRSETS (multiple response) sets.
Returns/takes a dictionary of the form:
--multiple category sets: {setName: {"setType": "C", "label": lbl,
"varNames": [<list_of_varNames>]}}
--multiple dichotomy sets: {setName: {"setType": "D", "label": lbl,
"varNames": [<list_of_varNames>], "countedValue": countedValue}}
--extended multiple dichotomy sets: {setName: {"setType": "E",
"label": lbl, "varNames": [<list_of_varNames>], "countedValue":
countedValue, 'firstVarIsLabel': <bool>}}
For example:
categorical = {"setType": "C", "label": "labelC",
"varNames": ["salary", "educ"]}
dichotomous1 = {"setType": "D", "label": "labelD",
"varNames": ["salary", "educ"], "countedValue": "Yes"}
dichotomous2 = {"setType": "D", "label": "", "varNames":
["salary", "educ", "jobcat"], "countedValue": "No"}
extended1 = {"setType": "E", "label": "", "varNames": ["mevar1",
"mevar2", "mevar3"], "countedValue": "1",
"firstVarIsLabel": True}
extended2 = {"setType": "E", "label":
"Enhanced set with user specified label", "varNames":
["mevar4", "mevar5", "mevar6"], "countedValue":
"Yes", "firstVarIsLabel": False}
multRespDefs = {"testSetC": categorical, "testSetD1": dichotomous1,
"testSetD2": dichotomous2, "testSetEx1": extended1,
"testSetEx2": extended2}
"""
if self.spssVersion[0] == 21:
print ("WARNING: getting multiple response definitions may not " +
"work with this IO version")
mrDefs = c_char_p()
func = self.spssio.spssGetMultRespDefs
retcode = func(c_int(self.fh), byref(mrDefs))
if retcode > 0:
msg = "Problem getting multiple response definitions"
raise SPSSIOError(msg, retcode)
multRespDefs = {}
if mrDefs.value:
for mrDef in mrDefs.value.split("\n"):
for setName, rest in self._getMultRespDef(mrDef).iteritems():
multRespDefs[setName] = rest
self.freeMemoryItems["spssFreeMultRespDefs Normal"] = (mrDefs,)
# I am not sure whether 'extended' MR definitions complement or replace
# 'normal' MR definitions. I assumed 'complement'.
mrDefsEx = c_char_p()
func = self.spssio.spssGetMultRespDefsEx
retcode = func(c_int(self.fh), byref(mrDefsEx))
if retcode > 0:
msg = "Problem getting extended multiple response definitions"
raise SPSSIOError(msg, retcode)
multRespDefsEx = {}
if mrDefsEx.value:
for mrDefEx in mrDefsEx.value.split("\n"):
for setName, rest in self._getMultRespDef(mrDefEx).iteritems():
multRespDefsEx[setName] = rest
self.freeMemoryItems["spssFreeMultRespDefs Extended"] = (mrDefsEx,)
multRespDefs.update(multRespDefsEx)
return multRespDefs
@multRespDefs.setter
def multRespDefs(self, multRespDefs):
if not multRespDefs:
return
normal = self._setMultRespDefs(multRespDefs)
extended = self._setMultRespDefsEx(multRespDefs)
if normal and extended:
combinedDefs = normal + " \n" + extended
elif normal and not extended:
combinedDefs = normal
elif extended and not normal:
combinedDefs = extended
func = self.spssio.spssSetMultRespDefs
retcode = func(c_int(self.fh), c_char_p(combinedDefs))
if retcode > 0:
msg = "Problem setting multiple response definitions"
raise SPSSIOError(msg, retcode)
@property
@decode
def caseWeightVar(self):
"""Get/Set WEIGHT variable.
Takes a valid varName, and returns weight variable, if any, as a
string."""
varNameBuff = create_string_buffer(65)
func = self.spssio.spssGetCaseWeightVar
retcode = func(c_int(self.fh), byref(varNameBuff))
if retcode > 0:
msg = "Problem getting case weight variable name"
raise SPSSIOError(msg, retcode)
return varNameBuff.value
@caseWeightVar.setter
def caseWeightVar(self, varName):
if not varName:
return
func = self.spssio.spssSetCaseWeightVar
retcode = func(c_int(self.fh), c_char_p(varName))
if retcode > 0:
msg = "Problem setting case weight variable name %r" % varName
raise SPSSIOError(msg, retcode)
@property
@decode
def dateVariables(self): # entirely untested!
"""Get/Set DATE information. This function reports the Forecasting
(Trends) date variable information, if any, in IBM SPSS Statistics
data files. Entirely untested and not implemented in reader/writer"""
nElements = c_int()
MAX_ARRAY_SIZE = 100 # or self.nCases?
dateInfoArr = (POINTER(c_long) * MAX_ARRAY_SIZE)()
func = self.spssio.spssGetDateVariables
retcode = func(c_int(self.fh), byref(nElements), byref(dateInfoArr))
if retcode > 0:
raise SPSSIOError("Error getting TRENDS information", retcode)
nElem = nElements.value
dateInfo = [dateInfoArr[0][i] for i in xrange(nElem)]
fixedDateInfo = dateInfo[:6]
otherDateInfo = [dateInfo[i: i + 3] for i in xrange(6, nElem, 3)]
self.freeMemoryItems["spssFreeDateVariables"] = (dateInfoArr,)
dateInfo = {"fixedDateInfo": fixedDateInfo,
"otherDateInfo": otherDateInfo}
return dateInfo
@dateVariables.setter
def dateVariables(self, dateInfo): # entirely untested!
dateInfo = dateInfo["fixedDateInfo"] + dateInfo["otherDateInfo"]
dateInfo = reduce(list.__add__, dateInfo) # flatten list
isAllFloats = all([isinstance(d, float) for d in dateInfo])
isSixPlusTriplets = (len(dateInfo) - 6) % 3 == 0
if not isAllFloats and isSixPlusTriplets:
msg = ("TRENDS date info must consist of 6 fixed elements" +
"+ <nCases> three-element groups of other date info " +
"(all floats)")
raise TypeError(msg)
func = self.spssio.spssSetDateVariables
dateInfoArr = (nElements * c_long)(*dateInfo)
retcode = func(c_int(self.fh), c_int(nElements), dateInfoArr)
if retcode > 0:
raise SPSSIOError("Error setting TRENDS information", retcode)
@property
@decode
def textInfo(self):
"""Get/Set text information.
Takes a savFileName and returns a string of the form: "File %r built
using SavReaderWriter.py version %s (%s)". This is akin to, but
*not* equivalent to the SPSS syntax command DISPLAY DOCUMENTS"""
textInfo = create_string_buffer(256)
retcode = self.spssio.spssGetTextInfo(c_int(self.fh), byref(textInfo))
if retcode > 0:
raise SPSSIOError("Error getting textInfo", retcode)
return textInfo.value
@textInfo.setter
def textInfo(self, savFileName):
info = (os.path.basename(savFileName), __version__, time.asctime())
textInfo = "File '%s' built using SavReaderWriter.py version %s (%s)"
textInfo = textInfo % info
if self.ioUtf8 and isinstance(savFileName, unicode):
textInfo = textInfo.encode("utf-8")
func = self.spssio.spssSetTextInfo
retcode = func(c_int(self.fh), c_char_p(textInfo[:256]))
if retcode > 0:
raise SPSSIOError("Error setting textInfo", retcode)
@property
def fileCompression(self):
"""Get/Set the file compression.
Returns/Takes a compression switch which may be any of the following:
'uncompressed', 'standard', or 'zlib'. Zlib comression requires SPSS
v21 I/O files."""
compression = {0: "uncompressed", 1: "standard", 2: "zlib"}
compSwitch = c_int()
func = self.spssio.spssGetCompression
retcode = func(c_int(self.fh), byref(compSwitch))
if retcode > 0:
raise SPSSIOError("Error getting file compression", retcode)
return compression.get(compSwitch.value)
@fileCompression.setter
def fileCompression(self, compSwitch):
compression = {"uncompressed": 0, "standard": 1, "zlib": 2}
compSwitch = compression.get(compSwitch)
func = self.spssio.spssSetCompression
retcode = func(c_int(self.fh), c_int(compSwitch))
invalidSwitch = retcodes.get(retcode) == 'SPSS_INVALID_COMPSW'
if invalidSwitch and self.spssVersion[0] < 21:
msg = "Writing zcompressed files requires >=v21 SPSS I/O libraries"
raise ValueError(msg)
elif retcode > 0:
raise SPSSIOError("Error setting file compression", retcode)
@property
@decode
def fileLabel(self):
"""Get/Set FILE LABEL (id string)
Takes a file label (basestring), and returns file label, if any, as
a string."""
idStr = create_string_buffer(65)
retcode = self.spssio.spssGetIdString(c_int(self.fh), byref(idStr))
if retcode > 0:
raise SPSSIOError("Error getting file label (id string)", retcode)
return idStr.value
@fileLabel.setter
def fileLabel(self, idStr):
if idStr is None:
idStr = "File created by user %r at %s"[:64] % \
(getpass.getuser(), time.asctime())
if self.ioUtf8 and isinstance(idStr, unicode):
idStr = idStr.encode("utf-8")
retcode = self.spssio.spssSetIdString(c_int(self.fh), c_char_p(idStr))
if retcode > 0:
raise SPSSIOError("Error setting file label (id string)", retcode)
###############################################################################
class SavHeaderReader(Header):
"""
This class contains methods that read the data dictionary of an SPSS
data file. This yields the same information as the Spss command 'DISPLAY
DICTIONARY' NB: do not confuse an Spss dictionary with a Python
dictionary!
Typical use:
with SavHeaderReader(savFileName) as spssDict:
wholeDict = spssDict.dataDictionary()
print unicode(spssDict)
"""
def __init__(self, savFileName, ioUtf8=False, ioLocale=None):
""" Constructor. Initializes all vars that can be recycled """
super(SavHeaderReader, self).__init__(savFileName, "rb", None,
ioUtf8, ioLocale)
self.fh = self.openSavFile()
self.varNames, self.varTypes = self.varNamesTypes
self.numVars = self.numberofVariables
self.nCases = self.numberofCases
def __str__(self):
""" This function returns a report of the SPSS data dictionary
(i.e., the header), in the encoding of the spss file"""
return unicode(self).encode(self.fileEncoding)
def __unicode__(self):
""" This function returns a report of the SPSS data dictionary
(i.e., the header)."""
report = ""
if self.textInfo:
report += self.textInfo + os.linesep
report += self.reportSpssDataDictionary(self.dataDictionary())
return report
def __enter__(self):
""" This function returns the DictionaryReader object itself so
its methods become available for use with context managers
('with' statements)."""
return self
def __exit__(self, type, value, tb):
""" This function closes the spss data file and does some cleaning."""
if type is not None:
pass # Exception occurred
self.close()
def close(self):
"""This function closes the spss data file and does some cleaning."""
if not segfaults:
self.freeMemory(self.freeMemoryItems)
self.closeSavFile(self.fh, mode="rb")
self.closeFile() # this in particular may cause errors in Linux
def dataDictionary(self):
""" This function returns all the dictionary items. It returns
a Python dictionary based on the Spss dictionary of the given
Spss file. This is equivalent to the Spss command 'DISPLAY
DICTIONARY'."""
items = ["varNames", "varTypes", "valueLabels", "varLabels",
"formats", "missingValues", "measureLevels",
"columnWidths", "alignments", "varSets", "varRoles",
"varAttributes", "fileAttributes", "fileLabel",
"multRespDefs", "caseWeightVar", "multRespDefs"]
if self.ioUtf8:
items = map(unicode, items)
dataDictionary = dict([(item, getattr(self, item)) for item in items])
return dataDictionary
def reportSpssDataDictionary(self, dataDict):
""" This function reports information from the Spss dictionary
of the active Spss dataset. The parameter 'dataDict' is the return
value of dataDictionary()"""
report = []
#import pprint
#pprint.pprint(dataDict)
for kwd, allValues in sorted(dataDict.items()):
report.append("#" + kwd.upper())
if hasattr(allValues, "items"):
for varName, values in allValues.iteritems():
if hasattr(values, "items"):
isList = kwd in ("missingValues", "multRespDefs")
for k, v in sorted(values.iteritems()):
if isList and isinstance(v, (list, tuple)):
vStr = map(unicode.lower, map(unicode, v))
report.append("%s: %s -- %s" %
(varName, k, ", ".join(vStr)))
else:
report.append("%s: %s -- %s" %
(varName, unicode(k).strip(), v))
else:
if isinstance(values, list):
entry = "%s -- %s" % (varName, ", ".join(values))
report.append(entry)
elif values != "":
report.append("%s -- %s" % (varName, values))
else:
if isinstance(allValues, basestring) and allValues:
allValues = [allValues]
for varName in allValues:
report.append(varName)
return os.linesep.join(report)
###############################################################################
class SavReader(Header):
""" Read Spss system files (.sav)
Parameters:
-savFileName: the file name of the spss data file
-returnHeader: Boolean that indicates whether the first record should
be a list of variable names (default = False)
-recodeSysmisTo: indicates to which value missing values should
be recoded (default = None),
-selectVars: indicates which variables in the file should be selected.
The variables should be specified as a list or a tuple of
valid variable names. If None is specified, all variables
in the file are used (default = None)
-idVar: indicates which variable in the file should be used for use as id
variable for the 'get' method (default = None)
-verbose: Boolean that indicates whether information about the spss data
file (e.g., number of cases, variable names, file size) should be
printed on the screen (default = False).
-rawMode: Boolean that indicates whether values should get SPSS-style
formatting, and whether date variables (if present) should be converted
to ISO-dates. If True, the program does not format any values, which
increases processing speed. (default = False)
-ioUtf8: indicates the mode in which text communicated to or from the
I/O Module will be. Valid values are True (UTF-8 mode aka Unicode mode)
and False (Codepage mode). Cf. SET UNICODE=ON/OFF (default = False)
-ioLocale: indicates the locale of the I/O module. Cf. SET LOCALE (default
= None, which corresponds to locale.getlocale()[0])
Typical use:
savFileName = "d:/someFile.sav"
with SavReader(savFileName, returnHeader=True) as sav:
header = sav.next()
for line in sav:
process(line)
"""
def __init__(self, savFileName, returnHeader=False, recodeSysmisTo=None,
verbose=False, selectVars=None, idVar=None, rawMode=False,
ioUtf8=False, ioLocale=None):
""" Constructor. Initializes all vars that can be recycled """
super(SavReader, self).__init__(savFileName, "rb", None,
ioUtf8, ioLocale)
self.savFileName = savFileName
self.returnHeader = returnHeader
self.recodeSysmisTo = recodeSysmisTo
self.verbose = verbose
self.selectVars = selectVars
self.idVar = idVar
self.rawMode = rawMode
self.header = self.getHeader(self.selectVars)
self.bareformats, self.varWids = self._splitformats()
self.autoRawMode = self._isAutoRawMode()
self.ioUtf8_ = ioUtf8
self.sysmis_ = self.sysmis
self.numVars = self.numberofVariables
self.nCases = self.numberofCases
self.myStruct = self.getStruct(self.varTypes, self.varNames)
self.unpack_from = self.myStruct.unpack_from
self.seekNextCase = self.spssio.spssSeekNextCase
self.caseBuffer = self.getCaseBuffer()
if psycoOk:
self._items = psyco.proxy(self._items) # 3 x faster!
def __enter__(self):
""" This function opens the spss data file (context manager)."""
if self.verbose and self.ioUtf8_:
print unicode(self).replace(os.linesep, "\n")
elif self.verbose:
print str(self).replace(os.linesep, "\n")
return iter(self)
def __exit__(self, type, value, tb):
""" This function closes the spss data file and does some cleaning."""
if type is not None:
pass # Exception occurred
self.close()
def close(self):
"""This function closes the spss data file and does some cleaning."""
if not segfaults:
self.freeMemory(self.freeMemoryItems)
self.closeSavFile(self.fh, mode="rb")
self.closeFile()
del self.spssio
def __len__(self):
""" This function reports the number of cases (rows) in the spss data
file. For example: len(SavReader(savFileName))"""
return self.nCases
def __cmp__(self, other):
""" This function implements behavior for all of the comparison
operators so comparisons can be made between SavReader instances,
or comparisons between SavReader instances and integers."""
if not isinstance(other, (SavReader, int)):
raise TypeError
other = other if isinstance(other, int) else len(other)
if len(self) < other:
return -1
elif len(self) == other:
return 0
else:
return 1
def __hash__(self):
"""This function returns a hash value for the object to ensure it
is hashable."""
return id(self)
def __str__(self):
"""This function returns a conscise file report of the spss data file
For example str(SavReader(savFileName))"""
return unicode(self).encode(self.fileEncoding)
def __unicode__(self):
"""This function returns a conscise file report of the spss data file,
For example unicode(SavReader(savFileName))"""
self.fileReport = self.getFileReport(self.savFileName, self.varNames,
self.varTypes, self.formats,
self.nCases)
return self.fileReport
def _isAutoRawMode(self):
"""Helper function for formatValues function. Determines whether
iterating over each individual value is really needed"""
hasDates = bool(set(self.bareformats.values()) & set(supportedDates))
hasNfmt = "N" in self.bareformats
hasRecodeSysmis = self.recodeSysmisTo is not None
items = [hasDates, hasNfmt, hasRecodeSysmis, self.ioUtf8_]
return False if any(items) else True
def formatValues(self, record):
"""This function formats date fields to ISO dates (yyyy-mm-dd), plus
some other date/time formats. The SPSS N format is formatted to a
character value with leading zeroes. System missing values are recoded
to <recodeSysmisTo>. If rawMode==True, this function does nothing"""
if self.rawMode or self.autoRawMode:
return record # 6-7 times faster!
for i, value in enumerate(record):
varName = self.header[i]
varType = self.varTypes[varName]
bareformat_ = self.bareformats[varName]
varWid = self.varWids[varName]
if varType == 0:
# recode system missing values, if present and desired
if value > self.sysmis_:
pass
else:
record[i] = self.recodeSysmisTo
# format N-type values (=numerical with leading zeroes)
if bareformat_ == "N":
#record[i] = str(value).zfill(varWid)
record[i] = "%%0%dd" % varWid % value #15 x faster (zfill)
# convert SPSS dates to ISO dates
elif bareformat_ in supportedDates:
fmt = supportedDates[bareformat_]
args = (value, fmt, self.recodeSysmisTo)
record[i] = self.spss2strDate(*args)
elif varType > 0:
value = value[:varType]
if self.ioUtf8_:
record[i] = value.decode("utf-8")
else:
record[i] = value
return record
def _items(self, start=0, stop=None, step=1, returnHeader=False):
""" This is a helper function to implement the __getitem__ and
the __iter__ special methods. """
if returnHeader:
yield self.header
if all([start == 0, stop is None, step == 1]): # used as iterator
retcode = 0
else:
retcode = self.seekNextCase(c_int(self.fh), c_long(0)) # reset
if stop is None:
stop = self.nCases
selection = self.selectVars is not None
for case in xrange(start, stop, step):
if start:
# only call this when iterating over part of the records
retcode = self.seekNextCase(c_int(self.fh), c_long(case))
if retcode > 0:
raise SPSSIOError("Error seeking case %d" % case, retcode)
elif stop == self.nCases:
self.printPctProgress(case, self.nCases)
record = self.record
if selection:
record = self.selector(record)
record = [record] if len(record) == 1 else list(record)
record = self.formatValues(record)
yield record
def __iter__(self):
"""This function allows the object to be used as an iterator"""
return self._items(0, None, 1, self.returnHeader)
def __getitem__(self, key):
""" This function reports the record of case number <key>.
For example: firstRecord = SavReader(savFileName)[0].
The <key> argument may also be a slice, for example:
firstfiveRecords = SavReader(savFileName)[:5].
You can also do stuff like (numpy required!):
savReader(savFileName)[1:5, 1]"""
is_slice = isinstance(key, slice)
is_array_slice = key is Ellipsis or isinstance(key, tuple)
if is_slice:
start, stop, step = key.indices(self.nCases)
elif is_array_slice:
return self._get_array_slice(key, self.nCases, len(self.header))
else:
key = operator.index(key)
start = key + self.nCases if key < 0 else key
if not 0 <= start < self.nCases:
raise IndexError("Index out of bounds")
stop = start + 1
step = 1
records = self._items(start, stop, step)
if is_slice:
return list(records)
return next(records)
def _get_array_slice(self, key, nRows, nCols):
"""This is a helper function to implement array slicing with numpy"""
if not numpyOk:
raise ImportError("Array slicing requires the numpy library")
is_index = False
rstart = cstart = 0
try:
row, col = key
if row < 0:
row = nRows + row
if col < 0:
col = nCols + col
## ... slices
if isinstance(row, slice) and col is Ellipsis:
# reader[1:2, ...]
rstart, rstop, rstep = row.indices(nRows)
cstart, cstop, cstep = 0, nRows, 1
elif row is Ellipsis and isinstance(col, slice):
# reader[..., 1:2]
rstart, rstop, rstep = 0, nRows, 1
cstart, cstop, cstep = col.indices(nCols)
elif isinstance(row, slice) and isinstance(col, slice):
# reader[1:2, 1:2]
rstart, rstop, rstep = row.indices(nRows)
cstart, cstop, cstep = col.indices(nCols)
elif row is Ellipsis and col is Ellipsis:
# reader[..., ...]
rstart, rstop, rstep = 0, nRows, 1
cstart, cstop, cstep = 0, nCols, 1
## ... indexes
elif isinstance(row, int) and col is Ellipsis:
# reader[1, ...]
rstart, rstop, rstep = row, row + 1, 1
cstart, cstop, cstep = 0, nCols, 1
is_index = True
elif row is Ellipsis and isinstance(col, int):
# reader[..., 1]
rstart, rstop, rstep = 0, nRows, 1
cstart, cstop, cstep = col, col + 1, 1
is_index = True
elif isinstance(row, int) and isinstance(col, int):
# reader[1, 1]
rstart, rstop, rstep = row, row + 1, 1
cstart, cstop, cstep = col, col + 1, 1
is_index = True
# ... slice + index
elif isinstance(row, slice) and isinstance(col, int):
# reader[1:2, 1]
rstart, rstop, rstep = row.indices(nRows)
cstart, cstop, cstep = col, col + 1, 1
elif isinstance(row, int) and isinstance(col, slice):
# reader[1, 1:2]
rstart, rstop, rstep = row, row + 1, 1
cstart, cstop, cstep = col.indices(nCols)
try:
if not 0 <= rstart < nRows:
raise IndexError("Index out of bounds")
if not 0 <= cstart < nCols:
raise IndexError("Index out of bounds")
key = (Ellipsis, slice(cstart, cstop, cstep))
except UnboundLocalError:
msg = "The array index is either invalid, or not implemented"
raise TypeError(msg)
except TypeError:
# reader[...]
rstart, rstop, rstep = 0, nRows, 1
key = (Ellipsis, Ellipsis)
records = self._items(rstart, rstop, rstep)
result = numpy.array(list(records))[key].tolist()
if is_index:
return result[0]
return result
def head(self, n=5):
""" This convenience function returns the first <n> records. """
return self[:abs(n)]
def tail(self, n=5):
""" This convenience function returns the last <n> records. """
return self[-abs(n):]
def __contains__(self, item):
""" This function implements membership testing and returns True if
<idVar> contains <item>. Thus, it requires the 'idVar' parameter to
be set. For example: reader = SavReader(savFileName, idVar="ssn")
"987654321" in reader """
return bool(self.get(item))
def get(self, key, default=None, full=False):
""" This function returns the records for which <idVar> == <key>
if <key> in <savFileName>, else <default>. Thus, the function mimics
dict.get, but note that dict[key] is NOT implemented. NB: Even though
this uses a binary search, this is not very fast on large data (esp.
the first call, and with full=True)
Parameters:
-key: key for which the corresponding record should be returned
-default: value that should be returned if <key> is not found
(default: None)
-full: value that indicates whether *all* records for which
<idVar> == <key> should be returned (default: False)
For example: reader = SavReader(savFileName, idVar="ssn")
reader.get("987654321", "social security number not found!")"""
if not self.idVar in self.varNames:
msg = ("SavReader object must be instantiated with an existing " +
"variable as an idVar argument")
raise NameError(msg)
#two slightly modified functions from the bisect module
def bisect_right(a, x, lo=0, hi=None):
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo + hi) // 2
if x < a[mid][0]:
hi = mid # a[mid][0], not a[mid]
else:
lo = mid + 1
return lo
def bisect_left(a, x, lo=0, hi=None):
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo + hi) // 2
if a[mid][0] < x:
lo = mid + 1 # a[mid][0], not a[mid]
else:
hi = mid
return lo
idPos = self.varNames.index(self.idVar)
if not hasattr(self, "isSorted"):
self.isSorted = True
if self.varTypes[self.idVar] == 0:
if not isinstance(key, (int, float)):
return default
self.recordz = ((record[idPos], i) for i,
record in enumerate(iter(self)))
else:
if not isinstance(key, basestring):
return default
self.recordz = ((record[idPos].rstrip(), i) for i,
record in enumerate(iter(self)))
self.recordz = sorted(self.recordz)
insertLPos = bisect_left(self.recordz, key)
insertRPos = bisect_right(self.recordz, key)
if full:
result = [self[record[1]] for record in
self.recordz[insertLPos: insertRPos]]
else:
if insertLPos == insertRPos:
return default
result = self[self.recordz[insertLPos][1]]
if result:
return result
return default
def getSavFileInfo(self):
""" This function reads and returns some basic information of the open
spss data file."""
return (self.numVars, self.nCases, self.varNames, self.varTypes,
self.formats, self.varLabels, self.valueLabels)
def memoize(f):
"""Memoization decorator
http://code.activestate.com/recipes/577219-minimalistic-memoization/"""
cache = {}
MAXCACHE = 10**4
def memf(*x):
if x not in cache:
cache[x] = f(*x)
if len(cache) > MAXCACHE:
cache.popitem()
return cache[x]
return memf
@memoize
def spss2strDate(self, spssDateValue, fmt, recodeSysmisTo):
"""This function converts internal SPSS dates (number of seconds
since midnight, Oct 14, 1582 (the beginning of the Gregorian calendar))
to a human-readable format"""
try:
if not hasattr(self, "gregorianEpoch"):
self.gregorianEpoch = datetime.datetime(1582, 10, 14, 0, 0, 0)
theDate = (self.gregorianEpoch +
datetime.timedelta(seconds=spssDateValue))
return datetime.datetime.strftime(theDate, fmt)
except OverflowError:
return recodeSysmisTo
except TypeError:
return recodeSysmisTo
except ValueError:
return recodeSysmisTo
def getFileReport(self, savFileName, varNames, varTypes,
formats, nCases):
""" This function prints a report about basic file characteristics """
bytes = os.path.getsize(savFileName)
kb = float(bytes) / 2**10
mb = float(bytes) / 2**20
(fileSize, label) = (mb, "MB") if mb > 1 else (kb, "kB")
systemString = self.systemString
spssVersion = ".".join(map(str, self.spssVersion))
lang, cp = locale.getlocale()
intEnc = "Utf-8/Unicode" if self.ioUtf8 else "Codepage (%s)" % cp
varlist = []
line = " %%0%sd. %%s (%%s - %%s)" % len(str(len(varNames) + 1))
for cnt, varName in enumerate(varNames):
lbl = "string" if varTypes[varName] > 0 else "numerical"
format_ = formats[varName]
varlist.append(line % (cnt + 1, varName, format_, lbl))
info = {"savFileName": savFileName,
"fileSize": fileSize,
"label": label,
"nCases": nCases,
"nCols": len(varNames),
"nValues": nCases * len(varNames),
"spssVersion": "%s (%s)" % (systemString, spssVersion),
"ioLocale": self.ioLocale,
"ioUtf8": intEnc,
"fileEncoding": self.fileEncoding,
"fileCodePage": self.fileCodePage,
"isCompatible": "Yes" if self.isCompatibleEncoding() else "No",
"local_language": lang,
"local_encoding": cp,
"varlist": os.linesep.join(varlist),
"sep": os.linesep,
"asterisks": 70 * "*"}
report = ("%(asterisks)s%(sep)s" +
"*File %(savFileName)r (%(fileSize)3.2f %(label)s) has " +
"%(nCols)s columns (variables) and %(nCases)s rows " +
"(%(nValues)s values)%(sep)s" +
"*The file was created with SPSS version: %(spssVersion)s%" +
"(sep)s" +
"*The interface locale is: %(ioLocale)r%(sep)s" +
"*The interface mode is: %(ioUtf8)s%(sep)s" +
"*The file encoding is: %(fileEncoding)r (Code page: " +
"%(fileCodePage)s)%(sep)s" +
"*File encoding and the interface encoding are compatible:" +
" %(isCompatible)s%(sep)s" +
"*Your computer's locale is: %(local_language)r (Code " +
"page: %(local_encoding)s)%(sep)s" +
"*The file contains the following variables:%(sep)s" +
"%(varlist)s%(sep)s%(asterisks)s%(sep)s")
return report % info
def getHeader(self, selectVars):
"""This function returns the variable names, or a selection thereof
(as specified as a list using the selectVars parameter), as a list."""
if selectVars is None:
header = self.varNames
elif isinstance(selectVars, (list, tuple)):
diff = set(selectVars).difference(set(self.varNames))
if diff:
msg = "Variable names misspecified (%r)" % ", ".join(diff)
raise NameError(msg)
varPos = [varNames.index(v) for v in self.varNames
if v in selectVars]
self.selector = operator.itemgetter(*varPos)
header = self.selector(self.varNames)
header = [header] if not isinstance(header, tuple) else header
else:
msg = ("Variable names list misspecified. Must be 'None' or a " +
"list or tuple of existing variables")
raise TypeError(msg)
return header
###############################################################################
class SavWriter(Header):
""" Write Spss system files (.sav)
Parameters:
* Formal
-savFileName: the file name of the spss data file. File names that end with
'.zsav' are compressed using the ZLIB (ZSAV) compression scheme
(requires v21 SPSS I/O files), while for file names that end with '.sav'
the 'old' compression scheme is used (it is not possible to generate
uncompressed files unless you modify the source code).
-varNames: list of the variable names in the order in which they appear in
in the spss data file.
-varTypes: varTypes dictionary {varName: varType}, where varType == 0 means
'numeric', and varType > 0 means 'character' of that length (in bytes)
* Optional (the associated SPSS commands are given in CAPS)
-valueLabels: value label dictionary {varName: {value: label}} Cf. VALUE
LABELS (default: None)
-varLabels: variable label dictionary {varName: varLabel}. Cf. VARIABLE
LABEL (default: None)
-formats: format_ dictionary {varName: printFmt} Cf. FORMATS
(default: None)
-missingValues: missing values dictionary {varName: {missing value spec}}
(default: None). Cf. MISSING VALUES. For example:
{'someNumvar1': {'values': [999, -1, -2]}, # discrete values
'someNumvar2': {'lower': -9, 'upper':-1}, # range, cf. -9 THRU -1
'someNumvar3': {'lower': -9, 'upper':-1, 'value': 999},
'someStrvar1': {'values': ['foo', 'bar', 'baz']},
'someStrvar2': {'values': 'bletch'}}
---The following three parameters must all three be set, if used---
-measureLevels: measurement level dictionary {varName: <level>}
Valid levels are: "unknown", "nominal", "ordinal", "scale",
"ratio", "flag", "typeless". Cf. VARIABLE LEVEL (default: None)
-columnWidths: column display width dictionary {varName: <int>}.
Cf. VARIABLE WIDTH. (default: None --> >= 10 [stringVars] or automatic
[numVars])
-alignments: alignment dictionary {varName: <left/center/right>}
Cf. VARIABLE ALIGNMENT (default: None --> left)
---
-varSets: sets dictionary {setName: list_of_valid_varNames}.
Cf. SETSMR command. (default: None)
-varRoles: variable roles dictionary {setName: list_of_valid_varNames}.
Cf. VARIABLE ROLE (default: None)
-varAttributes: variable attributes dictionary {varName: {attribName:
attribValue} (default: None). For example: varAttributes = {'gender':
{'Binary': 'Yes'}, 'educ': {'DemographicVars': '1'}}. Cf. VARIABLE
ATTRIBUTES. (default: None)
-fileAttributes: file attributes dictionary {attribName: attribValue}
For example: {'RevisionDate[1]': '10/29/2004', 'RevisionDate[2]':
'10/21/2005'}. Square brackets indicate attribute arrays, which must
start with 1. Cf. FILE ATTRIBUTES. (default: None)
-fileLabel: file label string, which defaults to "File created by user
<username> at <datetime>" is file label is None. Cf. FILE LABEL
(default: None)
-multRespDefs: Multiple response sets definitions (dichotomy groups or
category groups) dictionary {setName: <set definition>}. In SPSS syntax,
'setName' has a dollar prefix ('$someSet'). See also docstring of
multRespDefs method. Cf. MRSETS. (default: None)
-caseWeightVar: valid varName that is set as case weight (cf. WEIGHT BY)
-overwrite: Boolean that indicates whether an existing Spss file should be
overwritten (default: True)
-ioUtf8: indicates the mode in which text communicated to or from the
I/O Module will be. Valid values are True (UTF-8/unicode mode, cf. SET
UNICODE=ON) or False (Codepage mode, SET UNICODE=OFF) (default: False)
-ioLocale: indicates the locale of the I/O module, cf. SET LOCALE (default:
None, which is the same as locale.getlocale()[0])
-mode: indicates the mode in which <savFileName> should be opened. Possible
values are "wb" (write), "ab" (append), "cp" (copy: initialize header
using <refSavFileName> as a reference file, cf. APPLY DICTIONARY).
(default: "wb")
-refSavFileName: reference file that should be used to initialize the
header (aka the SPSS data dictionary) containing variable label, value
label, missing value, etc etc definitions. Only relevant in conjunction
with mode="cp". (default: None)
Typical use:
records = [['Test1', 1, 1], ['Test2', 2, 1]]
varNames = ['var1', 'v2', 'v3']
varTypes = {'var1': 5, 'v2': 0, 'v3': 0}
savFileName = "test.sav"
with SavWriter(savFileName, varNames, varTypes) as sav:
sav.writerows(records)
"""
def __init__(self, savFileName, varNames, varTypes, valueLabels=None,
varLabels=None, formats=None, missingValues=None,
measureLevels=None, columnWidths=None, alignments=None,
varSets=None, varRoles=None, varAttributes=None,
fileAttributes=None, fileLabel=None, multRespDefs=None,
caseWeightVar=None, overwrite=True, ioUtf8=False,
ioLocale=None, mode="wb", refSavFileName=None):
""" Constructor. Initializes all vars that can be recycled """
super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
self.savFileName = savFileName
self.varNames = self.encode(varNames)
self.varTypes = self.encode(varTypes)
self.overwrite = overwrite
self.mode = mode
self.refSavFileName = refSavFileName
self.fh = super(Header, self).openSavFile(self.savFileName, self.mode,
self.refSavFileName)
self.myStruct = self.getStruct(self.varTypes, self.varNames, self.mode)
self.pack_into = self.myStruct.pack_into
self.sysmis_ = self.sysmis
self.ioUtf8_ = ioUtf8
self.pad_8_lookup = self._getPaddingLookupTable(self.varTypes)
#self._getVarHandles()
if self.mode == "wb":
self.openWrite(self.savFileName, self.overwrite)
self.varNamesTypes = self.varNames, self.varTypes
self.valueLabels = valueLabels
self.varLabels = varLabels
self.formats = formats
self.missingValues = missingValues
self.measureLevels = measureLevels
self.columnWidths = columnWidths
self.alignments = alignments
self.varSets = varSets
self.varRoles = varRoles
self.varAttributes = varAttributes
self.fileAttributes = fileAttributes
self.fileLabel = fileLabel
self.multRespDefs = multRespDefs
self.caseWeightVar = caseWeightVar
triplet = [measureLevels, columnWidths, alignments]
if all([item is None for item in triplet]):
self._setColWidth10()
self.textInfo = self.savFileName
self.commitHeader()
self.caseBuffer = self.getCaseBuffer()
def __enter__(self):
"""This function returns the writer object itself so the writerow and
writerows methods become available for use with 'with' statements"""
return self
def __exit__(self, type, value, tb):
""" This function closes the spss data file."""
if type is not None:
pass # Exception occurred
self.closeSavFile(self.fh, mode="wb")
self.closeFile()
def _getVarHandles(self):
"""This function returns a handle for a variable, which can then be
used to read or write (depending on how the file was opened) values
of the variable. If handle is associated with an output file, the
dictionary must be written with spssCommitHeader before variable
handles can be obtained via spssGetVarHandle. Helper function for
__setitem__. *Not currently used*"""
varHandles = {}
varHandle = c_double()
func = self.spssio.spssGetVarHandle
for varName in self.varNames:
retcode = func(c_int(self.fh), c_char_p(varName), byref(varHandle))
varHandles[varName] = varHandle.value
if retcode > 0:
msg = "Error getting variable handle for variable %r"
raise SPSSIOError(msg % varName, retcode)
return varHandles
def __setitem__(self, varName, value):
"""This function sets the value of a variable for the current case.
The current case is not written out to the data file until
spssCommitCaseRecord is called. *Not currently used*, but it was just
begging to be implemented. ;-) For example:
self['someNumVar'] = 10
self['someStrVar'] = 'foo'"""
if not isinstance(value, (float, int, basestring)):
raise ValueError("Value %r has wrong type: %s" %
value, type(value))
varHandle = self.varHandles[varName]
if self.varTypes[varName] == 0:
funcN = self.spssio.spssSetValueNumeric
retcode = funcN(c_int(self.fh), c_double(varHandle),
c_double(value))
else:
funcC = self.spssio.spssSetValueChar
retcode = funcC(c_int(self.fh), c_double(varHandle),
c_char_p(value))
if retcode > 0:
isString = isinstance(value, basestring)
valType = "character" if isString else "numerical"
msg = "Error setting %s value %r for variable %r"
raise SPSSIOError(msg % (valType, value, varName), retcode)
def openWrite(self, savFileName, overwrite):
""" This function opens a file in preparation for creating a new IBM
SPSS Statistics data file"""
if os.path.exists(savFileName) and not os.access(savFileName, os.W_OK):
raise IOError("No write access for file %r" % savFileName)
if overwrite or not os.path.exists(savFileName):
# always compress files, either zsav or standard.
if savFileName.lower().endswith(".zsav"):
self.fileCompression = "zlib" # only with v21 libraries!
else:
self.fileCompression = "standard"
elif not overwrite and os.path.exists(savFileName):
raise IOError("File %r already exists!" % savFileName)
def convertDate(self, day, month, year):
"""This function converts a Gregorian date expressed as day-month-year
to the internal SPSS date format. The time portion of the date variable
is set to 0:00. To set the time portion if the date variable to another
value, use convertTime."""
d, m, y = c_int(day), c_int(month), c_int(year)
spssDate = c_double()
retcode = self.spssio.spssConvertDate(d, m, y, byref(spssDate))
if retcode > 0:
msg = "Problem converting date value '%s-%s-%s'" % (d, m, y)
raise SPSSIOError(msg, retcode)
return spssDate.value
def convertTime(self, day, hour, minute, second):
"""This function converts a time given as day, hours, minutes, and
seconds to the internal SPSS time format."""
d, h, m, s, spssTime = (c_int(day), c_int(hour), c_int(minute),
c_double(float(second)), c_double())
retcode = self.spssio.spssConvertTime(d, h, m, s, byref(spssTime))
if retcode > 0:
msg = ("Problem converting time value '%s %s:%s:%s'" %
(day, hour, minute, second))
raise SPSSIOError(msg, retcode)
return spssTime.value
def spssDateTime(self, datetimeStr="2001-12-08", strptimeFmt="%Y-%m-%d"):
""" This function converts a date/time string into an SPSS date,
using a strptime format."""
dt = time.strptime(datetimeStr, strptimeFmt)
day, month, year = dt.tm_mday, dt.tm_mon, dt.tm_year
hour, minute, second = dt.tm_hour, dt.tm_min, dt.tm_sec
return (self.convertDate(day, month, year) +
self.convertTime(0, hour, minute, second))
def commitHeader(self):
"""This function writes the data dictionary to the data file associated
with file handle 'fh'. Before any case data can be written, the
dictionary must be committed; once the dictionary has been committed,
no further changes can be made to it."""
retcode = self.spssio.spssCommitHeader(c_int(self.fh))
if retcode > 0:
raise SPSSIOError("Problem committing header", retcode)
def _getPaddingLookupTable(self, varTypes):
"""Helper function that returns a lookup table that maps string lengths
to string lengths to the nearest ceiled multiple of 8. For example:
{1:%-8s, 7:%-8s, 9: %-16s, 24: %-24s}. Purpose: Get rid of trailing
null bytes"""
strLenghts = varTypes.values()
strRange = xrange(min(strLenghts), max(strLenghts) + 1)
return dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strRange])
def writerow(self, record):
""" This function writes one record, which is a Python list."""
float_ = float
for i, value in enumerate(record):
varName = self.varNames[i]
varType = self.varTypes[varName]
if varType == 0:
try:
value = float_(value)
except ValueError:
value = self.sysmis_
else:
# Get rid of trailing null bytes --> 7 x faster than 'ljust'
value = self.pad_8_lookup[varType] % value
if self.ioUtf8_:
if isinstance(value, unicode):
value = value.encode("utf-8")
record[i] = value
self.record = record
def writerows(self, records):
""" This function writes all records."""
nCases = len(records)
for case, record in enumerate(records):
self.writerow(record)
self.printPctProgress(case, nCases)
###############################################################################
if __name__ == "__main__":
import contextlib
import csv
import collections
import pprint
import cProfile
import pstats
#savFileName = r"C:\Program Files\IBM\SPSS\Statistics\20\Samples\English\Employee data.sav"
## ---------------------------------
## SAV DATA DICTIONARY READER
## ---------------------------------
## ----- Typical use
with SavHeaderReader("employee data.sav") as spssDict:
print str(spssDict) # prints a report of the header information
wholeDict = spssDict.dataDictionary()
## ----- Copy header info from one file to another
savFileName = "employee data.sav"
with SavHeaderReader(savFileName) as spssDict:
wholeDict = spssDict.dataDictionary()
reader = SavReader(savFileName)
writer = SavWriter('test_wholeDict.sav', **wholeDict)
with contextlib.nested(reader, writer) as (r, w):
for line in r:
w.writerow(line)
## ---------------------------------
## SAV READER
## ---------------------------------
## ----- Typical use
savFileName = "employee data.sav"
with SavReader(savFileName, returnHeader=True) as sav:
header = sav.next()
for line in sav:
pass # do stuff
## ----- Explore the data file.
## ... Get some basic file info
reader = SavReader(savFileName, idVar="id")
(numVars, nCases, varNames, varTypes,
formats, varLabels, valueLabels) = reader.getSavFileInfo()
print "The file contains %d records" % len(reader)
print str(reader) # prints a file report
## ... Use indexing, slicing, or array slicing to access records
print "The first six records look like this\n", reader[:6]
print "The first record looks like this\n", reader[0]
print "The last four records look like this\n", reader.tail(4)
print "The first five records look like this\n", reader.head()
if numpyOK:
print "First column:\n", reader[..., 0]
print "Row 4 & 5, first three cols\n", reader[4:6, :3]
## ... Do a binary search for records --> idVar
print reader.get(4, "not found") # gets 1st record where id==4
print reader.get(4, "not found", full=True) # gets all records where id==4
## ... Use a named tuple to get specific values
record = collections.namedtuple("_", " ".join(varNames))(*reader[0])
print record.id, record.educ, record.salary, record.bdate
reader.close()
## ----- Convert file into .csv
savFileName = "in_unicode.sav"
csvFileName = "test_out.csv"
sav = SavReader(savFileName, ioLocale="dutch") # locale spec depends on OS
f = open(csvFileName, "wb")
with contextlib.nested(sav, f) as (sav, f):
writer = csv.writer(f)
for line in sav:
writer.writerow(line)
print "Done! Csv file written: %s" % f.name
## ---------------------------------
## SAV WRITER
## ---------------------------------
## ----- Write many rows
savFileName = "another_test.zsav" # .zsav = zlib compressed (--> v21 libs)
records = [['Test1', 1, 1, '2010-08-11'], ['Test2', 2, 1, '1910-01-12']]
varNames = ['var1', 'var2', 'var3', 'bdate']
varTypes = {'var1': 50, 'var2': 0, 'var3': 0, 'bdate': 10}
with SavWriter(savFileName, varNames, varTypes) as sav:
sav.writerows(records)
print "Done! %s" % sav.savFileName
savFileName = "employee data_OUT.sav"
varNames = ['id', 'gender', 'bdate', 'educ', 'jobcat', 'salary',
'salbegin', 'jobtime', 'prevexp', 'minority']
varLabels = {'gender': 'guys/gals'}
varTypes = {'salary': 0, 'jobcat': 0, 'bdate': 0, 'minority': 0,
'prevexp': 0, 'gender': 1, 'salbegin': 0, 'jobtime': 0,
'educ': 0, 'id': 0}
varSets = {'SALARY': ['salbegin', 'salary'],
'DEMOGR': ['gender', 'minority', 'educ']}
varAttributes = {'salary': {'DemographicVars': '1'},
'jobcat': {'DemographicVars': '1'},
'gender': {'Binary': 'Yes'},
'educ': {'DemographicVars': '1'}}
fileAttributes = {'TheDate[2]': '10/21/2005',
'RevisionDate[2]': '10/21/2005',
'RevisionDate[1]': '10/29/2004'}
missingValues = {'educ': {'values': [1, 2, 3]}, 'gender': {'values': 'x'}}
records = [[1.0, 'm ', 11654150400.0, 15.0, 3.0,
57000.0, 27000.0, 98.0, 144.0, 0.0],
[2.0, 'm ', 11852956800.0, 16.0, 1.0,
40200.0, 18750.0, 98.0, 36.0, 0.0]]
with SavWriter(savFileName, varNames, varTypes,
varLabels=varLabels, varSets=varSets,
missingValues=missingValues, varAttributes=varAttributes,
fileAttributes=fileAttributes) as sav:
sav.writerows(records)
print "Done! %s" % sav.savFileName
## ----- Write one row
# ... var1 is a 5-char string var, the others are numerical:
# ... formats, varLabels, valueLabels, missingValues etc. may also be
# ... None (default).
records = [['Test1', 1, 1, '2010-08-11'], ['Test2', 2, 1, '1910-01-12']]
varNames = ['var1', 'v2', 'v3', 'bdate']
varTypes = {'var1': 41, 'v2': 0, 'v3': 0, 'bdate': 0}
formats = {'var1': 'A41', 'v2': 'F3.1', 'v3': 'F5.1', 'bdate': 'EDATE40'}
missingValues = {'var1': {'values': ['Test1', 'Test2']},
'v2': {'values': 1}}
varLabels = {'var1': 'This is variable 1',
'v2': 'This is v2!',
'bdate': 'dob'}
valueLabels = {'var1': {'Test1': 'Test1 value label',
'Test2': 'Test2 value label'},
'v2': {1: 'yes', 2: 'no'}}
# This also shows how date fields can be converted into spss dates.
# Spss dates are *stored* as the number of seconds since Oct 14, 1582, but
# *displayed* as <format>. In this case they are displayed as EDATE
# (European date, cf. ADATE = American date), ie. as dd.mm.yyyy
savFileName = "test1.sav"
with SavWriter(savFileName, varNames, varTypes,
valueLabels, varLabels, formats) as sav:
pos = varNames.index("bdate")
for record in records:
record[pos] = sav.spssDateTime(record[pos], "%Y-%m-%d")
sav.writerow(record)
print "Done! %s" % sav.savFileName
## a test to check if non-ascii encodings work well
## source: http://www.omniglot.com/language/phrases/hello.htm
greetings = [
[0, 'Arabic', u'\u0627\u0644\u0633\u0644\u0627\u0645\u0020\u0639\u0644\u064a\u0643\u0645'],
[1, 'Assamese', u'\u09a8\u09ae\u09b8\u09cd\u0995\u09be\u09f0'],
[2, 'Bengali', u'\u0986\u09b8\u09b8\u09be\u09b2\u09be\u09ae\u09c1\u0986\u09b2\u09be\u0987\u0995\u09c1\u09ae'],
[3, 'English', u'Greetings and salutations'],
[4, 'Georgian', u'\u10d2\u10d0\u10db\u10d0\u10e0\u10ef\u10dd\u10d1\u10d0'],
[5, 'Kazakh', u'\u0421\u04d9\u043b\u0435\u043c\u0435\u0442\u0441\u0456\u0437 \u0431\u0435'],
[6, 'Russian', u'\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435'],
[7, 'Spanish', u'\xa1Hola!'],
[8, 'Swiss German', u'Gr\xfcezi'],
[9, 'Thai', u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'],
[10, 'Walloon', u'Bondjo\xfb'],
[11, 'Telugu', u'\u0c0f\u0c2e\u0c02\u0c21\u0c40'],
]
savFileName = "greetings_" + greetings[0][-1] + ".sav"
varNames = ['line', u'Bondjo\xfb', 'greeting']
varTypes = {'line': 0, u'Bondjo\xfb': 20, 'greeting': 50}
valueLabels = {u'Bondjo\xfb': {'Thai': u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'}}
missingValues = {'line': {'lower': 0, 'upper': 9}, u'Bondjo\xfb': {'values': u'\xa1Hola!'}}
varLabels = {'greeting': u'\u0627\u0644\u0633\u0644\u0627\u0645\u0020\u0639\u0644\u064a\u0643\u0645'}
with SavWriter(savFileName, varNames, varTypes, valueLabels, varLabels,
missingValues=missingValues, ioUtf8=True) as sav:
sav.writerows(greetings)
with SavReader(savFileName, ioUtf8=True)as sav:
for lino, line in enumerate(sav):
print lino, "%s -- %s" % (line[1].strip(), line[2]), line
with SavHeaderReader(savFileName, ioUtf8=True)as spssDict:
wholeDict = spssDict.dataDictionary()
#print wholeDict["varLabels"]["greeting"]
print unicode(spssDict)
pprint.pprint(wholeDict)
Diff to Previous Revision
--- revision 9 2012-08-14 14:45:25
+++ revision 10 2012-12-24 15:00:06
@@ -1,1090 +1,2868 @@
#!/usr/bin/env python
-# -*- coding: cp1252 -*-
-
-""" SavReaderWriter.py: A Python interface to the IBM SPSS Statistics Input Output Module
-(Linux: libspssdio.so.1, Mac: libspssdio.dylib, Windows: spssio32.dll). Read or Write SPSS system files (.sav) """
-
-# NOW WORKS WITH LINUX (Tested with Linux Ubuntu 10) AND MAC OS (Thanks to Rich Sadowsky!)
-# libspssdio.so.1, libspssdio.dylib, spssio32.dll + associated libaries and documentation can be downloaded here:
+# -*- coding: utf-8 -*-
+
+"""
+SavReaderWriter.py: A cross-platform Python interface to the IBM SPSS
+Statistics Input Output Module. Read or Write SPSS system files (.sav, .zsav)
+"""
+
+# libspssdio.so.1, libspssdio.dylib, spssio32.dll + associated libaries and
+# documentation can be downloaded here:
# https://www.ibm.com/developerworks/mydeveloperworks/wikis/home/wiki/We70df3195ec8_4f95_9773_42e448fa9029/page/Downloads%20for%20IBM%C2%AE%20SPSS%C2%AE%20Statistics?lang=en
-# NOTE: This supersedes SavReader.py
-# NOTE: Added SavDataDictionaryReader/Writer classes to read/write the Spss data file header
# ANY FEEDBACK ON THIS CODE IS WELCOME: "@".join(["fomcl", "yahoo.com"])
-# Mac support added Oct-22-2011 by Rich Sadowsky "@".join(["rich", "richsad.com"])
-
-from __future__ import with_statement # only Python 2.5
+# Mac support added Oct-22-2011: by Rich Sadowsky
+# "@".join(["rich", "richsad.com"])
+
+## December 2012:
+# -Added support for slicing, indexing, array slicing + other special methods
+# -Added support for writing spss date fields
+# -Added support for almost all meta data (missing values, sets, roles, etc.)
+# -Added support for 64 bit Windows (tested with Win7) and other OSs
+# (z/Linux, Solaris, HP Linux, IBM AIX (untested though)
+# -Added support for reading and writing zlib compressed (.zsav) files
+# -Removed pesky segfault error when freeing memory
+# -Removed errors related to encoding
+# -Removed error that prevented files originating from big-endian systems
+# being read properly in little-endian systems (and vice versa)
+# -Changed some Reader defaults (verbose=False, returnHeader=False)
+# -Renamed SavDataDictionaryReader into SavHeaderReader
+##
+
+from ctypes import *
+import ctypes.util
+import struct
import sys
+import platform
import os
import re
-import ctypes
-import struct
import operator
import math
import locale
import datetime
import time
-import tempfile
import getpass
-
+import encodings
+import functools
try:
import psyco
- psyco.full()
+ psycoOk = True
except ImportError:
- print "NOTE. Psyco module not found. Install this module to increase program performance"
-
-__author__ = 'Albert-Jan Roskam'
-__version__ = '3.0.0'
-
-retcodes = {0: 'SPSS_OK',
- 1: 'SPSS_FILE_OERROR',
- 2: 'SPSS_FILE_WERROR',
- 3: 'SPSS_FILE_RERROR',
- 4: 'SPSS_FITAB_FULL',
- 5: 'SPSS_INVALID_HANDLE',
- 6: 'SPSS_INVALID_FILE',
- 7: 'SPSS_NO_MEMORY',
- 8: 'SPSS_OPEN_RDMODE',
- 9: 'SPSS_OPEN_WRMODE',
- 10: 'SPSS_INVALID_VARNAME',
- 11: 'SPSS_DICT_EMPTY',
- 12: 'SPSS_VAR_NOTFOUND',
- 13: 'SPSS_DUP_VAR',
- 14: 'SPSS_NUME_EXP',
- 15: 'SPSS_STR_EXP',
- 16: 'SPSS_SHORTSTR_EXP',
- 17: 'SPSS_INVALID_VARTYPE',
- 18: 'SPSS_INVALID_MISSFOR',
- 19: 'SPSS_INVALID_COMPSW',
- 20: 'SPSS_INVALID_PRFOR',
- 21: 'SPSS_INVALID_WRFOR',
- 22: 'SPSS_INVALID_DATE',
- 23: 'SPSS_INVALID_TIME',
- 24: 'SPSS_NO_VARIABLES',
- 27: 'SPSS_DUP_VALUE',
- 28: 'SPSS_INVALID_CASEWGT',
- 30: 'SPSS_DICT_COMMIT',
- 31: 'SPSS_DICT_NOTCOMMIT',
- 33: 'SPSS_NO_TYPE2',
- 41: 'SPSS_NO_TYPE73',
- 45: 'SPSS_INVALID_DATEINFO',
- 46: 'SPSS_NO_TYPE999',
- 47: 'SPSS_EXC_STRVALUE',
- 48: 'SPSS_CANNOT_FREE',
- 49: 'SPSS_BUFFER_SHORT',
- 50: 'SPSS_INVALID_CASE',
- 51: 'SPSS_INTERNAL_VLABS',
- 52: 'SPSS_INCOMPAT_APPEND',
- 53: 'SPSS_INTERNAL_D_A',
- 54: 'SPSS_FILE_BADTEMP',
- 55: 'SPSS_DEW_NOFIRST',
- 56: 'SPSS_INVALID_MEASURELEVEL',
- 57: 'SPSS_INVALID_7SUBTYPE',
- 70: 'SPSS_INVALID_MRSETDEF',
- 71: 'SPSS_INVALID_MRSETNAME',
- 72: 'SPSS_DUP_MRSETNAME',
- -15: 'SPSS_EMPTY_DEW',
- -14: 'SPSS_NO_DEW',
- -13: 'SPSS_EMPTY_MULTRESP',
- -12: 'SPSS_NO_MULTRESP',
- -11: 'SPSS_NO_DATEINFO',
- -10: 'SPSS_NO_CASEWGT',
- -9: 'SPSS_NO_LABEL',
- -8: 'SPSS_NO_LABELS',
- -7: 'SPSS_EMPTY_VARSETS',
- -6: 'SPSS_NO_VARSETS',
- -5: 'SPSS_FILE_END',
- -4: 'SPSS_EXC_VALLABEL',
- -3: 'SPSS_EXC_LEN120',
- -2: 'SPSS_EXC_VARLABEL',
- -1: 'SPSS_EXC_LEN64'}
-
-printTypes = {1: ('SPSS_FMT_A', 'Alphanumeric'),
- 2: ('SPSS_FMT_AHEX', 'Alphanumeric hexadecimal'),
- 3: ('SPSS_FMT_COMMA', 'F Format with commas'),
- 4: ('SPSS_FMT_DOLLAR', 'Commas and floating dollar sign'),
- 5: ('SPSS_FMT_F', 'Default Numeric Format'),
- 6: ('SPSS_FMT_IB', 'Integer binary'),
- 7: ('SPSS_FMT_PIBHEX', 'Positive integer binary - hex'),
- 8: ('SPSS_FMT_P', 'Packed decimal'),
- 9: ('SPSS_FMT_PIB', 'Positive integer binary unsigned'),
- 10: ('SPSS_FMT_PK', 'Positive integer binary unsigned'),
- 11: ('SPSS_FMT_RB', 'Floating point binary'),
- 12: ('SPSS_FMT_RBHEX', 'Floating point binary hex'),
- 15: ('SPSS_FMT_Z', 'Zoned decimal'),
- 16: ('SPSS_FMT_N', 'N Format- unsigned with leading 0s'),
- 17: ('SPSS_FMT_E', 'E Format- with explicit power of 10'),
- 20: ('SPSS_FMT_DATE', 'Date format dd-mmm-yyyy'),
- 21: ('SPSS_FMT_TIME', 'Time format hh:mm:ss.s'),
- 22: ('SPSS_FMT_DATE_TIME', 'Date and Time'),
- 23: ('SPSS_FMT_ADATE', 'Date format dd-mmm-yyyy'),
- 24: ('SPSS_FMT_JDATE', 'Julian date - yyyyddd'),
- 25: ('SPSS_FMT_DTIME', 'Date-time dd hh:mm:ss.s'),
- 26: ('SPSS_FMT_WKDAY', 'Day of the week'),
- 27: ('SPSS_FMT_MONTH', 'Month'),
- 28: ('SPSS_FMT_MOYR', 'mmm yyyy'),
- 29: ('SPSS_FMT_QYR', 'q Q yyyy'),
- 30: ('SPSS_FMT_WKYR', 'ww WK yyyy'),
- 31: ('SPSS_FMT_PCT', 'Percent - F followed by %'),
- 32: ('SPSS_FMT_DOT', 'Like COMMA, switching dot for comma'),
- 33: ('SPSS_FMT_CCA', 'User Programmable currency format'),
- 34: ('SPSS_FMT_CCB', 'User Programmable currency format'),
- 35: ('SPSS_FMT_CCC', 'User Programmable currency format'),
- 36: ('SPSS_FMT_CCD', 'User Programmable currency format'),
- 37: ('SPSS_FMT_CCE', 'User Programmable currency format'),
- 38: ('SPSS_FMT_EDATE','Date in dd/mm/yyyy style'),
- 39: ('SPSS_FMT_SDATE', 'Date in yyyy/mm/dd style')}
-
-reversePrintTypes = dict([(fmt[0][9:], value)
- for fmt, value in dict(zip(printTypes.values(),
- printTypes.keys())).iteritems()])
-
-######################################################################################
+ print ("NOTE. Psyco module not found. Install this module " +
+ "to increase program performance")
+ psycoOk = False
+try:
+ import numpy
+ numpyOk = True
+except ImportError:
+ print ("NOTE. Numpy module not found. Install this module " +
+ "to use array slicing")
+ numpyOk = False
+
+# change this to 'True' in case you experience segmentation
+# faults related to freeing memory.
+segfaults = False
+
+__author__ = "Albert-Jan Roskam" + " " + "@".join(["fomcl", "yahoo.com"])
+__version__ = "3.1.0"
+
+retcodes = {
+ 0: "SPSS_OK",
+ 1: "SPSS_FILE_OERROR",
+ 2: "SPSS_FILE_WERROR",
+ 3: "SPSS_FILE_RERROR",
+ 4: "SPSS_FITAB_FULL",
+ 5: "SPSS_INVALID_HANDLE",
+ 6: "SPSS_INVALID_FILE",
+ 7: "SPSS_NO_MEMORY",
+ 8: "SPSS_OPEN_RDMODE",
+ 9: "SPSS_OPEN_WRMODE",
+ 10: "SPSS_INVALID_VARNAME",
+ 11: "SPSS_DICT_EMPTY",
+ 12: "SPSS_VAR_NOTFOUND",
+ 13: "SPSS_DUP_VAR",
+ 14: "SPSS_NUME_EXP",
+ 15: "SPSS_STR_EXP",
+ 16: "SPSS_SHORTSTR_EXP",
+ 17: "SPSS_INVALID_VARTYPE",
+ 18: "SPSS_INVALID_MISSFOR",
+ 19: "SPSS_INVALID_COMPSW",
+ 20: "SPSS_INVALID_PRFOR",
+ 21: "SPSS_INVALID_WRFOR",
+ 22: "SPSS_INVALID_DATE",
+ 23: "SPSS_INVALID_TIME",
+ 24: "SPSS_NO_VARIABLES",
+ 27: "SPSS_DUP_VALUE",
+ 28: "SPSS_INVALID_CASEWGT",
+ 30: "SPSS_DICT_COMMIT",
+ 31: "SPSS_DICT_NOTCOMMIT",
+ 33: "SPSS_NO_TYPE2",
+ 41: "SPSS_NO_TYPE73",
+ 45: "SPSS_INVALID_DATEINFO",
+ 46: "SPSS_NO_TYPE999",
+ 47: "SPSS_EXC_STRVALUE",
+ 48: "SPSS_CANNOT_FREE",
+ 49: "SPSS_BUFFER_SHORT",
+ 50: "SPSS_INVALID_CASE",
+ 51: "SPSS_INTERNAL_VLABS",
+ 52: "SPSS_INCOMPAT_APPEND",
+ 53: "SPSS_INTERNAL_D_A",
+ 54: "SPSS_FILE_BADTEMP",
+ 55: "SPSS_DEW_NOFIRST",
+ 56: "SPSS_INVALID_MEASURELEVEL",
+ 57: "SPSS_INVALID_7SUBTYPE",
+ 58: "SPSS_INVALID_VARHANDLE",
+ 59: "SPSS_INVALID_ENCODING",
+ 60: "SPSS_FILES_OPEN",
+ 70: "SPSS_INVALID_MRSETDEF",
+ 71: "SPSS_INVALID_MRSETNAME",
+ 72: "SPSS_DUP_MRSETNAME",
+ 73: "SPSS_BAD_EXTENSION",
+ 74: "SPSS_INVALID_EXTENDEDSTRING",
+ 75: "SPSS_INVALID_ATTRNAME",
+ 76: "SPSS_INVALID_ATTRDEF",
+ 77: "SPSS_INVALID_MRSETINDEX",
+ 78: "SPSS_INVALID_VARSETDEF",
+ 79: "SPSS_INVALID_ROLE",
+
+ -15: "SPSS_EMPTY_DEW",
+ -14: "SPSS_NO_DEW",
+ -13: "SPSS_EMPTY_MULTRESP",
+ -12: "SPSS_NO_MULTRESP",
+ -11: "SPSS_NO_DATEINFO",
+ -10: "SPSS_NO_CASEWGT",
+ -9: "SPSS_NO_LABEL",
+ -8: "SPSS_NO_LABELS",
+ -7: "SPSS_EMPTY_VARSETS",
+ -6: "SPSS_NO_VARSETS",
+ -5: "SPSS_FILE_END",
+ -4: "SPSS_EXC_VALLABEL",
+ -3: "SPSS_EXC_LEN120",
+ -2: "SPSS_EXC_VARLABEL",
+ -1: "SPSS_EXC_LEN64"}
+
+allFormats = {
+ 1: ("SPSS_FMT_A", "Alphanumeric"),
+ 2: ("SPSS_FMT_AHEX", "Alphanumeric hexadecimal"),
+ 3: ("SPSS_FMT_COMMA", "F Format with commas"),
+ 4: ("SPSS_FMT_DOLLAR", "Commas and floating dollar sign"),
+ 5: ("SPSS_FMT_F", "Default Numeric Format"),
+ 6: ("SPSS_FMT_IB", "Integer binary"),
+ 7: ("SPSS_FMT_PIBHEX", "Positive integer binary - hex"),
+ 8: ("SPSS_FMT_P", "Packed decimal"),
+ 9: ("SPSS_FMT_PIB", "Positive integer binary unsigned"),
+ 10: ("SPSS_FMT_PK", "Positive integer binary unsigned"),
+ 11: ("SPSS_FMT_RB", "Floating point binary"),
+ 12: ("SPSS_FMT_RBHEX", "Floating point binary hex"),
+ 15: ("SPSS_FMT_Z", "Zoned decimal"),
+ 16: ("SPSS_FMT_N", "N Format- unsigned with leading 0s"),
+ 17: ("SPSS_FMT_E", "E Format- with explicit power of 10"),
+ 20: ("SPSS_FMT_DATE", "Date format dd-mmm-yyyy"),
+ 21: ("SPSS_FMT_TIME", "Time format hh:mm:ss.s"),
+ 22: ("SPSS_FMT_DATE_TIME", "Date and Time"),
+ 23: ("SPSS_FMT_ADATE", "Date format dd-mmm-yyyy"),
+ 24: ("SPSS_FMT_JDATE", "Julian date - yyyyddd"),
+ 25: ("SPSS_FMT_DTIME", "Date-time dd hh:mm:ss.s"),
+ 26: ("SPSS_FMT_WKDAY", "Day of the week"),
+ 27: ("SPSS_FMT_MONTH", "Month"),
+ 28: ("SPSS_FMT_MOYR", "mmm yyyy"),
+ 29: ("SPSS_FMT_QYR", "q Q yyyy"),
+ 30: ("SPSS_FMT_WKYR", "ww WK yyyy"),
+ 31: ("SPSS_FMT_PCT", "Percent - F followed by %"),
+ 32: ("SPSS_FMT_DOT", "Like COMMA, switching dot for comma"),
+ 33: ("SPSS_FMT_CCA", "User Programmable currency format"),
+ 34: ("SPSS_FMT_CCB", "User Programmable currency format"),
+ 35: ("SPSS_FMT_CCC", "User Programmable currency format"),
+ 36: ("SPSS_FMT_CCD", "User Programmable currency format"),
+ 37: ("SPSS_FMT_CCE", "User Programmable currency format"),
+ 38: ("SPSS_FMT_EDATE", "Date in dd/mm/yyyy style"),
+ 39: ("SPSS_FMT_SDATE", "Date in yyyy/mm/dd style")}
+
+MAXLENGTHS = {
+ "SPSS_MAX_VARNAME": (64, "Variable name"),
+ "SPSS_MAX_SHORTVARNAME": (8, "Short (compatibility) variable name"),
+ "SPSS_MAX_SHORTSTRING": (8, "Short string variable"),
+ "SPSS_MAX_IDSTRING": (64, "File label string"),
+ "SPSS_MAX_LONGSTRING": (32767, "Long string variable"),
+ "SPSS_MAX_VALLABEL": (120, "Value label"),
+ "SPSS_MAX_VARLABEL": (256, "Variable label"),
+ "SPSS_MAX_7SUBTYPE": (40, "Maximum record 7 subtype"),
+ "SPSS_MAX_ENCODING": (64, "Maximum encoding text")}
+
+supportedDates = { # uses ISO dates wherever applicable.
+ "DATE": "%Y-%m-%d",
+ "JDATE": "%Y-%m-%d",
+ "EDATE": "%Y-%m-%d",
+ "SDATE": "%Y-%m-%d",
+ "DATETIME": "%Y-%m-%d %H:%M:%S",
+ "WKDAY": "%A %H:%M:%S",
+ "ADATE": "%Y-%m-%d",
+ "WKDAY": "%A",
+ "MONTH": "%B",
+ "MOYR": "%B %Y",
+ "WKYR": "%W WK %Y"}
+
+userMissingValues = {
+ "SPSS_NO_MISSVAL": 0,
+ "SPSS_ONE_MISSVAL": 1,
+ "SPSS_TWO_MISSVAL": 2,
+ "SPSS_THREE_MISSVAL": 3,
+ "SPSS_MISS_RANGE": -2,
+ "SPSS_MISS_RANGEANDVAL": -3}
+###############################################################################
+
+
+class SPSSIOError(Exception):
+ """
+ Error class for the IBM SPSS Statistics Input Output Module
+ """
+
+ def __init__(self, msg=u"Unknown", retcode=None):
+ if isinstance(retcode, int):
+ code = retcodes.get(retcode, retcode)
+ elif isinstance(retcode, list):
+ code = ", ".join([retcodes.get(retcodex) for retcodex in retcode])
+ else:
+ code = "unknown code %r" % retcode
+ msg = "%s [retcode: %r]" % (msg, code)
+ Exception.__init__(self, msg)
+###############################################################################
+
+
+def wide2utf8(fn):
+ """Take a unicode file name string and encode it to a multibyte string
+ that Windows can use to represent file names (CP65001, UTF-8)
+ http://msdn.microsoft.com/en-us/library/windows/desktop/dd374130"""
+
+ from ctypes import wintypes
+
+ _CP_UTF8 = 65001
+ _CP_ACP = 0 # ANSI
+ _LPBOOL = POINTER(c_long)
+
+ _wideCharToMultiByte = windll.kernel32.WideCharToMultiByte
+ _wideCharToMultiByte.restype = c_int
+ _wideCharToMultiByte.argtypes = [wintypes.UINT, wintypes.DWORD,
+ wintypes.LPCWSTR, c_int, wintypes.LPSTR, c_int, wintypes.LPCSTR, _LPBOOL]
+
+ codePage = _CP_UTF8
+ dwFlags = 0
+ lpWideCharStr = fn
+ cchWideChar = len(fn)
+ lpMultiByteStr = None
+ cbMultiByte = 0 # zero requests size
+ lpDefaultChar = None
+ lpUsedDefaultChar = None
+
+ # get size
+ mbcssize = _wideCharToMultiByte(
+ codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
+ cbMultiByte, lpDefaultChar, lpUsedDefaultChar)
+ if mbcssize <= 0:
+ raise WinError(mbcssize)
+ lpMultiByteStr = create_string_buffer(mbcssize)
+
+ # convert
+ retcode = _wideCharToMultiByte(
+ codePage, dwFlags, lpWideCharStr, cchWideChar, lpMultiByteStr,
+ mbcssize, lpDefaultChar, lpUsedDefaultChar)
+ if retcode <= 0:
+ raise WinError(retcode)
+ return lpMultiByteStr.value
+
+###############################################################################
+
class Generic(object):
-
- def __init__(self):
- self.spssio, self.fopen = self.loadLibrary()
-
+ """
+ Class for methods and data used in reading as well as writing
+ IBM SPSS Statistics data files
+ """
+
+ def __init__(self, savFileName, ioUtf8=False, ioLocale=None):
+ """Constructor. Note that interface locale and encoding can only
+ be set once"""
+ locale.setlocale(locale.LC_ALL, "")
+ self.savFileName = savFileName
+ self.libc = cdll.LoadLibrary(ctypes.util.find_library("c"))
+ self.spssio = self.loadLibrary()
+
+ #self.myStruct = self.getStruct(self.varTypes, self.varNames)
+ #self.caseBuffer = self.getCaseBuffer()
+
+ self.wholeCaseIn = self.spssio.spssWholeCaseIn
+ #self.unpack_from = self.myStruct.unpack_from
+ self.wholeCaseOut = self.spssio.spssWholeCaseOut
+ #self.pack_into = self.myStruct.pack_into
+
+ self.encoding_and_locale_set = False
+ if not self.encoding_and_locale_set:
+ self.encoding_and_locale_set = True
+ self.ioLocale = ioLocale
+ self.ioUtf8 = ioUtf8
+
+ def _encodeFileName(self, fn):
+ """Helper function to encode unicode file names into bytestring file
+ names encoded in the file system's encoding. Needed for C functions
+ that have a c_char_p filename argument.
+ http://effbot.org/pyref/sys.getfilesystemencoding.htm
+ http://docs.python.org/2/howto/unicode.html under 'unicode filenames'"""
+ if not isinstance(fn, unicode):
+ return fn
+ elif sys.platform.startswith("win"):
+ return wide2utf8(fn)
+ else:
+ encoding = sys.getfilesystemencoding()
+ encoding = "utf-8" if not encoding else encoding # actually, ascii
+ try:
+ return fn.encode(encoding)
+ except UnicodeEncodeError, e:
+ msg = ("File system encoding %r can not be used to " +
+ "encode file name %r [%s]")
+ raise ValueError(msg % (encoding, fn, e))
+
def loadLibrary(self):
- """ This function loads the spss I/O file (.dll or .so file or dylib file) and opens
- the spss data file for reading."""
- platform = sys.platform.lower()
- if platform.startswith("win"):
- try:
- os.environ["PATH"] += ";" + os.path.abspath(".")
- spssio = ctypes.windll.spssio32
- libc = ctypes.cdll.msvcrt
- fopen = libc._fdopen # libc.fopen() won't work on windows
- except WindowsError, e:
- msg = "Cannot find spssio32.dll in '%s'.\n" % os.path.abspath(".") + \
- "Py file and Dll should live in the same directory [%s]." % e
- raise Exception, msg
- return spssio, fopen
- elif platform.startswith("linux"):
- # add library search path to LD_LIBRARY_PATH environment variable
- # Type this in the terminal **before** running the program:
- # export LD_LIBRARY_PATH=/path/of/additional/sofiles
- # Then type: python nameOfWrapperToTheSpssReaderWriter.py
- if __name__ == "__main__":
- msg = "This program cannot be run directly. " + \
- "Type in the terminal: export LD_LIBRARY_PATH=/path/of/libspssdio " + \
- "Then type: python nameOfWrapperToTheSpssReaderWriter.py"
- raise Exception, msg
- spssio = ctypes.CDLL("libspssdio.so.1")
- libc = ctypes.CDLL("libc.so.6")
- fopen = libc.fdopen
- return spssio, fopen
- elif platform.startswith("darwin"):
- # you must put all the dylib files that come with the IBM
- # SPSS_Statistics_InputOutput_Modules_* package in the macos
- # directory somewhere that OS X can find them
- # one simple way to accomplish this is to copy them to /usr/lib
- spssio = ctypes.CDLL("libspssdio.dylib")
- libc = ctypes.CDLL("libc.dylib")
- fopen = libc.fdopen
- return spssio, fopen
+ """This function loads and returns the SPSSIO libraries,
+ depending on the platform."""
+ path = os.path.abspath(".")
+ is_32bit = platform.architecture()[0] == "32bit"
+ pf = sys.platform.lower()
+ load = WinDLL if pf.startswith("win") else CDLL
+ if pf.startswith("win"):
+ os.environ["PATH"] += ";" + path
+ spssio = load("spssio32") if is_32bit else load("spssio64")
+ elif pf.startswith("lin"): # most linux flavours, incl zLinux
+ spssio = load("libspssdio.so.1")
+ elif pf.startswith("darwin") or pf.startswith("mac"):
+ spssio = load("libspssdio.dylib")
+ elif pf.startswith("aix") and not is_32bit:
+ spssio = load("libspssdio.so.1")
+ elif pf.startswith("hp-ux"):
+ spssio = load("libspssdio.sl.1")
+ elif pf.startswith("sunos") and not is_32bit:
+ spssio = load("libspssdio.so.1")
else:
- msg = "Your platform ('%s') is not supported" % platform
- raise NotImplementedError, msg
-
- def loadSavFile(self, savFileName, spssio):
- """ This function opens IBM SPSS Statistics data files for reading and returns a handle that
- should be used for subsequent operations on the file. """
- if os.path.exists(savFileName):
- fh = self.fopen(savFileName, "rb")
- fhPtr = ctypes.byref(ctypes.c_int(fh))
- retcode = spssio.spssOpenRead(ctypes.c_char_p(savFileName), fhPtr)
- return retcode, fh
+ msg = "Your platform (%r) is not supported" % pf
+ raise NotImplementedError(msg)
+ return spssio
+
+ def errcheck(self, res, func, args):
+ """This function checks for errors during the execution of
+ function <func>"""
+ if not res:
+ msg = "Error performing %r operation on file %r."
+ raise IOError(msg % (func.__name__, self.savFileName))
+ return res
+
+ def openSavFile(self, savFileName, mode="rb", refSavFileName=None):
+ """This function opens IBM SPSS Statistics data files in mode <mode>
+ and returns a handle that should be used for subsequent operations on
+ the file. If <savFileName> is opened in mode "cp", meta data
+ information aka the spss dictionary is copied from <refSavFileName>"""
+ savFileName = os.path.abspath(savFileName) # fdopen wants full name
+ try:
+ fdopen = self.libc._fdopen # Windows
+ except AttributeError:
+ fdopen = self.libc.fdopen # Linux and others
+ fdopen.argtypes = [c_int, c_char_p]
+ fdopen.restype = c_void_p
+ fdopen.errcheck = self.errcheck
+ with open(savFileName, mode) as f:
+ self.fd = fdopen(f.fileno(), mode)
+ if mode == "rb":
+ spssOpen = self.spssio.spssOpenRead
+ elif mode == "wb":
+ spssOpen = self.spssio.spssOpenWrite
+ elif mode == "cp":
+ spssOpen = self.spssio.spssOpenWriteCopy
+ elif mode == "ab":
+ spssOpen = self.spssio.spssOpenAppend
+
+ savFileName = self._encodeFileName(savFileName)
+ refSavFileName = self._encodeFileName(refSavFileName)
+ sav = c_char_p(savFileName)
+ fh = c_int(self.fd)
+ if mode == "cp":
+ retcode = spssOpen(sav, c_char_p(refSavFileName), pointer(fh))
else:
- raise Exception, "File '%s' does not exist!" % savFileName
-
- def conversionFormatCtoPy(self, varTypes, varNames):
- """ This function generates a struct format string for the conversion
- between C and Python values. SPSS data files are assumed to have either
- 8-byte doubles/floats or n-byte chars[]/strings, where n is always
- 8 bytes or a multiple thereof."""
- structFmt = ""
- if sys.byteorder == "little":
- endianness = "<"
- elif sys.byteorder == "big":
- endianness = ">"
- else:
- endianness = "@"
- structFmt += endianness
+ retcode = spssOpen(sav, pointer(fh))
+
+ if retcode > 0:
+ msg = "Error opening file %r in mode %r"
+ raise SPSSIOError(msg % (savFileName, mode), retcode)
+ return fh.value
+
+ def closeSavFile(self, fh, mode="rb"):
+ """This function closes the sav file associated with <fh> that was open
+ in mode <mode>."""
+ if mode == "rb":
+ spssClose = self.spssio.spssCloseRead
+ elif mode == "wb":
+ spssClose = self.spssio.spssCloseWrite
+ elif mode == "ab":
+ spssClose = self.spssio.spssCloseAppend
+ retcode = spssClose(c_int(fh))
+ if retcode > 0:
+ raise SPSSIOError("Error closing file in mode %r" % mode, retcode)
+
+ def closeFile(self):
+ """Close file descriptor"""
+ try:
+ if self.fd: # return value is not null
+ self.libc.fclose.errcheck = self.errcheck
+ self.libc.fclose(c_void_p(self.fd)) # error in Linux?
+ else:
+ try:
+ # Windows
+ self.libc._close.errcheck = self.errcheck
+ self.libc._close(c_void_p(self.fd))
+ except AttributeError:
+ # Linux
+ self.libc.close.errcheck = self.errcheck
+ self.libc.close(c_void_p(self.fd))
+ except EnvironmentError, e:
+ print e
+
+ @property
+ def releaseInfo(self):
+ """This function reports release- and machine-specific information
+ about the open file."""
+ relInfo = ["release number", "release subnumber", "fixpack number",
+ "machine code", "floating-point representation code",
+ "compression scheme code", "big/little-endian code",
+ "character representation code"]
+ relInfoArr = (c_int * len(relInfo))()
+ retcode = self.spssio.spssGetReleaseInfo(c_int(self.fh), relInfoArr)
+ if retcode > 0:
+ raise SPSSIOError("Error getting ReleaseInfo", retcode)
+ info = dict([(item, relInfoArr[i]) for i, item in enumerate(relInfo)])
+ return info
+
+ @property
+ def spssVersion(self):
+ """Return the SPSS version as a three-tuple indicating major,
+ minor, and fixpack version as ints. NB: in the transition from
+ SPSS to IBM, a new four-digit versioning nomenclature is used.
+ This function uses the old three-digit nomenclature. Therefore,
+ no patch version information is available."""
+ info = self.releaseInfo
+ major = info["release number"]
+ minor = info["release subnumber"]
+ fixpack = info["fixpack number"]
+ return major, minor, fixpack
+
+ @property
+ def systemString(self):
+ """This function returns the name of the system under which the file
+ was created aa a string."""
+ sysName = create_string_buffer(42)
+ func = self.spssio.spssGetSystemString
+ retcode = func(c_int(self.fh), byref(sysName))
+ if retcode > 0:
+ raise SPSSIOError("Error getting SystemString", retcode)
+ return sysName.value
+
+ def getStruct(self, varTypes, varNames, mode="rb"):
+ """This function returns a compiled struct object. The required
+ struct format string for the conversion between C and Python
+ is created on the basis of varType and byte order.
+ --varTypes: SPSS data files have either 8-byte doubles/floats or n-byte
+ chars[]/ strings, where n is always 8 bytes or a multiple thereof.
+ --byte order: files are written in the byte order of the host system
+ (mode="wb") and read/appended using the byte order information
+ contained in the SPSS data file (mode is "ab" or "rb" or "cp")"""
+ if mode in ("ab", "rb", "cp"): # derive endianness from file
+ endianness = self.releaseInfo["big/little-endian code"]
+ endianness = ">" if endianness > 0 else "<"
+ elif mode == "wb": # derive endianness from host
+ if sys.byteorder == "little":
+ endianness = "<"
+ elif sys.byteorder == "big":
+ endianness = ">"
+ else:
+ endianness = "@"
+ structFmt = [endianness]
+ ceil = math.ceil
for varName in varNames:
varType = varTypes[varName]
if varType == 0:
- structFmt += "d"
+ structFmt.append("d")
else:
- fmt = str(int(math.ceil(int(varType) / 8.0) * 8))
- structFmt += fmt + "s"
- return structFmt
-
- def getSystemSysmisVal(self, spssio):
+ fmt = str(int(ceil(int(varType) / 8.0) * 8))
+ structFmt.append(fmt + "s")
+ return struct.Struct("".join(structFmt))
+
+ def getCaseBuffer(self):
+ """This function returns a buffer and a pointer to that buffer. A whole
+ case will be read into this buffer."""
+ caseSize = c_long()
+ retcode = self.spssio.spssGetCaseSize(c_int(self.fh), byref(caseSize))
+ caseBuffer = create_string_buffer(caseSize.value)
+ if retcode > 0:
+ raise SPSSIOError("Problem getting case buffer", retcode)
+ return caseBuffer
+
+ @property
+ def sysmis(self):
"""This function returns the IBM SPSS Statistics system-missing
- value for the host system."""
+ value ($SYSMIS) for the host system (also called 'NA' in other
+ systems)."""
try:
- sysmis = -1 * sys.float_info[0] # Python 2.6 and higher.
+ sysmis = -1 * sys.float_info[0] # Python 2.6 and higher.
except AttributeError:
- spssio.spssSysmisVal.restype = ctypes.c_float
- sysmis = spssio.spssSysmisVal()
+ self.spssio.spssSysmisVal.restype = c_float
+ sysmis = self.spssio.spssSysmisVal()
return sysmis
- def setInterfaceEncoding(self, spssio, interfaceEncoding):
- """This function sets the current interface encoding."""
- self.icodes = {"UTF-8": 0, "CODEPAGE": 1}
- interfaceEncoding = interfaceEncoding.upper()
- if interfaceEncoding not in self.icodes.keys():
- msg = "Invalid interface encoding ('%s'), valid values are 'UTF-8' or 'CODEPAGE'" % \
- interfaceEncoding
- raise Exception, msg
+ @property
+ def missingValuesLowHigh(self):
+ """This function returns the 'lowest' and 'highest' values used for
+ numeric missing value ranges on the host system. It may be called at
+ any time."""
+ lowest, highest = c_double(), c_double()
+ func = self.spssio.spssLowHighVal
+ retcode = func(byref(lowest), byref(highest))
+ return lowest.value, highest.value
+
+ @property
+ def ioLocale(self):
+ """This function gets/sets the I/O Module's locale.
+ This corresponds with the SPSS command SET LOCALE. The I/O Module's
+ locale is separate from that of the client application. The
+ <localeName> parameter and the return value are identical to those
+ for the C run-time function setlocale. The exact locale name
+ specification depends on the OS of the host sytem, but has the
+ following form:
+ <lang>_<territory>.<codeset>[@<modifiers>]
+ The 'codeset' and 'modifier' components are optional and in Windows,
+ aliases (e.g. 'english') may be used. When the I/O Module is first
+ loaded, its locale is set to the system default. See also:
+ --https://wiki.archlinux.org/index.php/Locale
+ --http://msdn.microsoft.com/en-us/library/39cwe7zf(v=vs.80).aspx"""
+ if hasattr(self, "setLocale"):
+ return self.setLocale
+ else:
+ currLocale = ".".join(locale.getlocale())
+ print "NOTE. Locale not set; getting current locale: ", currLocale
+ return currLocale
+
+ @ioLocale.setter
+ def ioLocale(self, localeName=""):
+ if not localeName:
+ localeName = ".".join(locale.getlocale())
+ func = self.spssio.spssSetLocale
+ func.restype = c_char_p
+ self.setLocale = func(c_int(locale.LC_ALL), c_char_p(localeName))
+ if self.setLocale is None:
+ raise ValueError("Invalid ioLocale: %r" % localeName)
+ return self.setLocale
+
+ @property
+ def fileCodePage(self):
+ """This function provides the Windows code page number of the encoding
+ applicable to a file."""
+ nCodePage = c_int()
+ func = self.spssio.spssGetFileCodePage
+ retcode = func(c_int(self.fh), byref(nCodePage))
+ return nCodePage.value
+
+ def isCompatibleEncoding(self):
+ """This function determines whether the file and interface encoding
+ are compatible."""
try:
- retcode = spssio.spssSetInterfaceEncoding(ctypes.c_int(self.icodes[interfaceEncoding]))
- return retcode
+ # Windows, note typo 'Endoding'!
+ func = self.spssio.spssIsCompatibleEndoding
except AttributeError:
- print "NOTE. Function 'setInterfaceEncoding' not found. You are using a .dll from SPSS < v16"
+ func = self.spssio.spssIsCompatibleEncoding
+ func.restype = c_bool
+ isCompatible = c_int()
+ retcode = func(c_int(self.fh), byref(isCompatible))
+ if retcode > 0:
+ msg = "Error testing encoding compatibility: %r"
+ raise SPSSIOError(msg % isCompatible.value, retcode)
+ if not isCompatible.value and not self.ioUtf8:
+ msg = ("NOTE. SPSS Statistics data file %r is written in a " +
+ "character encoding (%s) incompatible with the current " +
+ "ioLocale setting. It may not be readable. Consider " +
+ "changing ioLocale or setting ioUtf8=True.")
+ print msg % (self.savFileName, self.fileEncoding)
+ return bool(isCompatible.value)
+
+ @property
+ def ioUtf8(self):
+ """This function returns/sets the current interface encoding.
+ ioUtf8 = False --> CODEPAGE mode,
+ ioUtf8 = True --> UTF-8 mode, aka. Unicode mode
+ This corresponds with the SPSS command SHOW UNICODE (getter)
+ and SET UNICODE=ON/OFF (setter)."""
+ if hasattr(self, "ioUtf8_"):
+ return self.ioUtf8_
+ self.ioUtf8_ = self.spssio.spssGetInterfaceEncoding()
+ return bool(self.ioUtf8_)
+
+ @ioUtf8.setter
+ def ioUtf8(self, ioUtf8):
+ try:
+ retcode = self.spssio.spssSetInterfaceEncoding(c_int(int(ioUtf8)))
+ if retcode > 0 and not self.encoding_and_locale_set:
+ # not self.encoding_and_locale_set --> nested context managers
+ raise SPSSIOError("Error setting IO interface", retcode)
+ except TypeError:
+ msg = "Invalid interface encoding: %r (must be bool)"
+ raise SPSSIOError(msg % ioUtf8)
+
+ @property
+ def fileEncoding(self):
+ """This function obtains the encoding applicable to a file.
+ The encoding is returned as an IANA encoding name, such as
+ ISO-8859-1, which is then converted to the corresponding Python
+ codec name. If the file contains no file encoding, the locale's
+ preferred encoding is returned"""
+ preferredEncoding = locale.getpreferredencoding()
+ try:
+ pszEncoding = create_string_buffer(20) # is 20 enough??
+ func = self.spssio.spssGetFileEncoding
+ retcode = func(c_int(self.fh), byref(pszEncoding))
+ if retcode > 0:
+ raise SPSSIOError("Error getting file encoding", retcode)
+ iana_codes = encodings.aliases.aliases
+ rawEncoding = pszEncoding.value.lower()
+ if rawEncoding.replace("-", "") in iana_codes:
+ iana_code = rawEncoding.replace("-", "")
+ else:
+ iana_code = rawEncoding.replace("-", "_")
+ fileEncoding = iana_codes[iana_code]
+ return fileEncoding
+ except AttributeError:
+ print ("NOTE. Function 'getFileEncoding' not found. You are " +
+ "using a .dll from SPSS < v16.")
+ return preferredEncoding
+ except KeyError:
+ print ("NOTE. IANA coding lookup error. Code %r " % iana_code +
+ "does not map to any Python codec.")
+ return preferredEncoding
+
+ @property
+ def record(self):
+ """Get/Set a whole record from/to a pre-allocated buffer"""
+ retcode = self.wholeCaseIn(c_int(self.fh),
+ byref(self.caseBuffer))
+ if retcode > 0:
+ raise SPSSIOError("Problem reading row", retcode)
+ record = list(self.unpack_from(self.caseBuffer))
+ return record
+
+ @record.setter
+ def record(self, record):
+ try:
+ self.pack_into(self.caseBuffer, 0, *record)
+ except struct.error, e:
+ msg = "Use ioUtf8=True to write unicode strings [%s]" % e
+ raise TypeError(msg)
+ retcode = self.wholeCaseOut(c_int(self.fh),
+ c_char_p(self.caseBuffer.raw))
+ if retcode > 0:
+ raise SPSSIOError("Problem writing row:\n" + record, retcode)
+
+ def printPctProgress(self, nominator, denominator):
+ """This function prints the % progress when reading and writing
+ files"""
+ if nominator and nominator % 10**4 == 0:
+ pctProgress = (float(nominator) / denominator) * 100
+ print "%2.1f%%... " % pctProgress,
+###############################################################################
+
+
+class Header(Generic):
+
+ """
+ This class contains methods responsible for getting and setting meta data
+ that is embedded in the IBM SPSS Statistics data file. In SPSS speak, this
+ header information is known as the SPSS Data Dictionary (which has diddly
+ squat to do with a Python dictionary!).
+ """
+
+ def __init__(self, savFileName, mode, refSavFileName, ioUtf8, ioLocale=None):
+ """Constructor"""
+ super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
+ self.freeMemoryItems = {}
+ self.spssio = self.loadLibrary()
+ self.fh = super(Header, self).openSavFile(savFileName, mode,
+ refSavFileName)
+ self.varNames, self.varTypes = self.varNamesTypes
+ self.vNames = dict(zip(self.varNames, self.encode(self.varNames)))
+
+ def openSavFile(self):
+ """This function returns the file handle that was opened in the
+ super class"""
+ return self.fh
+
+ def decode(func):
+ """Decorator to Utf-8 decode all str items contained in a dictionary
+ If ioUtf8=True, the dictionary's keys and values are decoded, but only
+ values that are strs, lists, or dicts. For example:
+ {'v1': {'y': 'yy', 'z': 666}}-->{u'v1': {u'y': u'yy', u'z': 666}}"""
+ uS = lambda x: x.decode("utf-8") if isinstance(x, str) else x
+ uL = lambda x: map(uS, x) if isinstance(x, list) else x
+
+ @functools.wraps(func)
+ def wrapper(arg):
+ result = func(arg)
+ if not arg.ioUtf8:
+ return result # unchanged
+ if isinstance(result, str):
+ return uS(result)
+ utf8ifiedDict = {}
+ for k, v in result.iteritems():
+ k, v = uS(k), uS(v)
+ if isinstance(v, list):
+ v = map(uS, v)
+ elif isinstance(v, dict):
+ v = dict([(uS(x), uL(uS(y))) for x, y in v.items()])
+ utf8ifiedDict[k] = v
+ return utf8ifiedDict
+ return wrapper
+
+ def encode(self, item):
+ """Counter part of decode helper function, does the opposite of that
+ function (but is not a decorator)"""
+ if not self.ioUtf8:
+ return item # unchanged
+ utf8dify = lambda x: x.encode("utf-8") if isinstance(x, unicode) else x
+ if isinstance(item, list):
+ return map(utf8dify, item)
+ elif isinstance(item, dict):
+ return dict([(utf8dify(x), utf8dify(y)) for x, y in item.items()])
+ return utf8dify(item)
+
+ def freeMemory(self, items):
+ """Clean up: free memory claimed by e.g. getValueLabels and
+ getVarNamesTypes"""
+ # Previously crashed under other systems than WinXP (eg. Linux & Win7)
+ # This function actually has made my receding hairline worse ;-)
+ retcodex = []
+ isWindows = sys.platform.startswith("win")
+ pfVersion = platform.version().split(".")[0]
+ pfVersion = int(pfVersion) if pfVersion.isdigit() else 5 # 5 = WinXP
+ for funcName, args in sorted(items.iteritems()):
+ func = getattr(self.spssio, funcName.split()[0])
+ if isWindows and pfVersion <= 5:
+ retcode = func(*args)
+ retcodex.append(retcode)
+ else: # Linux Ubuntu and Windows 7, probably more
+ try:
+ # free arrays element-by-element
+ array1, array2, num = args
+ for element1, element2 in zip(array1, array2):
+ retcode = func(element1, element2, num)
+ retcodex.append(retcode)
+ except ValueError:
+ retcode = func(*args)
+ retcodex.append(retcode)
+ okay_or_not = "NOT okay!!!" if retcode > 0 else "okay"
+ print "...freeing %s --> %s" % (funcName[8:], okay_or_not)
+ retcode = max(retcodex)
+ if retcode > 0:
+ raise SPSSIOError("Error freeing memory", retcode)
+
+ @property
+ def numberofCases(self):
+ """This function reports the number of cases present in a data file.
+ Prehistoric files (< SPSS v6.0) don't contain nCases info, therefore
+ a guesstimate of the number of cases is given for those files"""
+ nCases = c_long()
+ func = self.spssio.spssGetNumberofCases
+ retcode = func(c_int(self.fh), byref(nCases))
+ if nCases.value == -1:
+ func = self.spssio.spssGetEstimatedNofCases
+ retcode = func(c_int(self.fh), byref(nCases))
+ if retcode > 0:
+ raise SPSSIOError("Error getting number of cases", retcode)
+ return nCases.value
+
+ @property
+ def numberofVariables(self):
+ """This function returns the number of variables (columns) in the
+ spss dataset"""
+ numVars = c_int()
+ func = self.spssio.spssGetNumberofVariables
+ retcode = func(c_int(self.fh), byref(numVars))
+ if retcode > 0:
+ raise SPSSIOError("Error getting number of variables", retcode)
+ return numVars.value
+
+ @property
+ def varNamesTypes(self):
+ """Get/Set variable names and types.
+ --Variable names is a list of the form ['var1', var2', 'etc']
+ --Variable types is a dictionary of the form {varName: varType}
+ The variable type code is an integer in the range 0-32767, 0
+ indicating a numeric variable (F8.2) and a positive value
+ indicating a string variable of that size (in bytes)."""
+ if hasattr(self, "varNames"):
+ return self.varNames, self.varTypes
+ numVars = self.numberofVariables
+ numVars_ = c_int(numVars)
+ varNamesArr = (POINTER(c_char_p) * numVars)()
+ varTypesArr = (POINTER(c_int) * numVars)()
+ func = self.spssio.spssGetVarNames
+ retcode = func(c_int(self.fh), byref(numVars_),
+ byref(varNamesArr), byref(varTypesArr))
+ if retcode > 0:
+ raise SPSSIOError("Error getting variable names & types", retcode)
+ varNames = [varNamesArr[0][i] for i in xrange(numVars)]
+ varTypes = [varTypesArr[0][i] for i in xrange(numVars)]
+ if self.ioUtf8:
+ varNames = [varName.decode("utf-8") for varName in varNames]
+ funcStr = "spssFreeVarNames"
+ self.freeMemoryItems[funcStr] = (varNamesArr, varTypesArr, numVars)
+ return varNames, dict(zip(varNames, varTypes))
+
+ @varNamesTypes.setter
+ def varNamesTypes(self, varNamesVarTypes):
+ badLengthMsg = "Empty or longer than %s chars" % \
+ (MAXLENGTHS['SPSS_MAX_VARNAME'][0])
+ varNames, varTypes = varNamesVarTypes
+ varNameRetcodes = {
+ 0: ('SPSS_NAME_OK', 'Valid standard name'),
+ 1: ('SPSS_NAME_SCRATCH', 'Valid scratch var name'),
+ 2: ('SPSS_NAME_SYSTEM' 'Valid system var name'),
+ 3: ('SPSS_NAME_BADLTH', badLengthMsg),
+ 4: ('SPSS_NAME_BADCHAR', 'Invalid character or embedded blank'),
+ 5: ('SPSS_NAME_RESERVED', 'Name is a reserved word'),
+ 6: ('SPSS_NAME_BADFIRST', 'Invalid initial char (otherwise OK)')}
+ validate = self.spssio.spssValidateVarname
+ func = self.spssio.spssSetVarName
+ for varName in self.varNames:
+ varLength = self.varTypes[varName]
+ retcode = validate(c_char_p(varName))
+ if retcode > 0:
+ msg = "%r is an invalid variable name [%r]" % \
+ (varName, ": ".join(varNameRetcodes.get(retcode)))
+ raise SPSSIOError(msg, retcode)
+ retcode = func(c_int(self.fh), c_char_p(varName), c_int(varLength))
+ if retcode > 0:
+ msg = "Problem setting variable name %r" % varName
+ raise SPSSIOError(msg, retcode)
+
+ @property
+ @decode
+ def valueLabels(self):
+ """Get/Set VALUE LABELS.
+ Takes a dictionary of the form {varName: {value: valueLabel}:
+ --{'numGender': {1: 'female', {2: 'male'}}
+ --{'strGender': {'f': 'female', 'm': 'male'}}"""
+ MAX_ARRAY_SIZE = 1000 # assume never more than 1000 labels
+ numLabels, valueLabels = c_int(), {}
+ for varName in self.varNames:
+ labelsArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
+ if self.varTypes[varName] == 0:
+ valuesArr = (POINTER(c_double) * MAX_ARRAY_SIZE)()
+ func = self.spssio.spssGetVarNValueLabels
+ funcStr = "spssFreeVarNValueLabels"
+ items = (valuesArr, labelsArr, numLabels)
+ self.freeMemoryItems[funcStr] = items
+ else:
+ valuesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
+ func = self.spssio.spssGetVarCValueLabels
+ funcStr = "spssFreeVarCValueLabels"
+ items = (valuesArr, labelsArr, numLabels)
+ self.freeMemoryItems[funcStr] = items
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName), byref(valuesArr),
+ byref(labelsArr), byref(numLabels))
+ if retcode > 0:
+ msg = "Error getting value labels of variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+ values = [valuesArr[0][i] for i in xrange(numLabels.value)]
+ labels = [labelsArr[0][i] for i in xrange(numLabels.value)]
+ valueLabelsX = [(val, lbl) for val, lbl in zip(values, labels)]
+ valueLabels[varName] = dict(valueLabelsX)
+ return valueLabels
+
+ @valueLabels.setter
+ def valueLabels(self, valueLabels):
+ if not valueLabels:
+ return
+ valLabN = self.spssio.spssSetVarNValueLabel
+ valLabC = self.spssio.spssSetVarCValueLabel
+ valueLabels = self.encode(valueLabels)
+ for varName, valueLabelsX in valueLabels.iteritems():
+ valueLabelsX = self.encode(valueLabelsX)
+ for value, label in valueLabelsX.iteritems():
+ if self.varTypes[varName] == 0:
+ retcode = valLabN(c_int(self.fh), c_char_p(varName),
+ c_double(value), c_char_p(label))
+ else:
+ retcode = valLabC(c_int(self.fh), c_char_p(varName),
+ c_char_p(value), c_char_p(label))
+ if retcode > 0:
+ msg = "Problem with setting value labels of variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+
+ @property
+ @decode
+ def varLabels(self):
+ """Get/set VARIABLE LABELS.
+ Returns/takes a dictionary of the form {varName: varLabel}.
+ For example: varLabels = {'salary': 'Salary (dollars)',
+ 'educ': 'Educational level (years)'}"""
+ SPSS_VARLABEL_SHORT = 120 # fixed amount
+ funcS = self.spssio.spssGetVarLabel
+ funcL = self.spssio.spssGetVarLabelLong
+ varLabels = {}
+ for varName in self.varNames:
+ varLabel = create_string_buffer(SPSS_VARLABEL_SHORT + 1)
+ vName = self.vNames[varName]
+ retcode = funcS(c_int(self.fh), c_char_p(vName), byref(varLabel))
+ if varLabel.value and len(varLabel.value) > SPSS_VARLABEL_SHORT:
+ lenBuff = MAXLENGTHS['SPSS_MAX_VARLABEL']
+ varLabel = create_string_buffer(lenBuff)
+ retcode = funcL(c_int(self.fh), c_char_p(varName),
+ byref(varLabel), c_int(lenBuff), byref(c_int))
+ varLabels[varName] = varLabel.value
+ if retcode > 0:
+ msg = "Error getting variable label of variable %r" % varName
+ raise SPSSIOError(msg, retcode)
+ return varLabels
+
+ @varLabels.setter
+ def varLabels(self, varLabels):
+ if not varLabels:
+ return
+ func = self.spssio.spssSetVarLabel
+ varLabels = self.encode(varLabels)
+ for varName, varLabel in varLabels.iteritems():
+ retcode = func(c_int(self.fh), c_char_p(varName),
+ c_char_p(varLabel))
+ if retcode > 0:
+ msg = ("Problem with setting variable label %r of variable %r"
+ % (varLabel, varName))
+ raise SPSSIOError(msg, retcode)
+
+ @property
+ @decode
+ def formats(self):
+ """Get the PRINT FORMATS, set PRINT and WRITE FORMATS.
+ Returns/takes a dictionary of the form {varName: <format_>.
+ For example: formats = {'salary': 'DOLLAR8', 'gender': 'A1',
+ 'educ': 'F8.2'}"""
+ if hasattr(self, "varformats"):
+ return self.varformats
+ printFormat_, printDec_, printWid_ = c_int(), c_int(), c_int()
+ func = self.spssio.spssGetVarPrintFormat
+ formats = {}
+ for varName in self.varNames:
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(printFormat_), byref(printDec_),
+ byref(printWid_))
+ if retcode > 0:
+ msg = "Error getting print format for variable %r"
+ raise SPSSIOError(msg % vName, retcode)
+ printFormat = allFormats.get(printFormat_.value)[0]
+ printFormat = printFormat.split("_")[-1]
+ format_ = printFormat + str(printWid_.value)
+ if self.varTypes[varName] == 0:
+ format_ += ("." + str(printDec_.value))
+ if format_.endswith(".0"):
+ format_ = format_[:-2]
+ formats[varName] = format_
+ return formats
+
+ def _splitformats(self):
+ """This function returns the 'bare' formats + variable widths,
+ e.g. format_ F5.3 is returned as 'F' and '5'"""
+ regex = re.compile("\w+(?P<varWid>\d+)[.]?\d?", re.I)
+ bareformats, varWids = {}, {}
+ for varName, format_ in self.formats.iteritems():
+ bareformats[varName] = re.sub(r"\d+.?", "", format_)
+ varWids[varName] = int(regex.search(format_).group("varWid"))
+ return bareformats, varWids
+
+ @formats.setter
+ def formats(self, formats):
+ if not formats:
+ return
+ reverseFormats = dict([(v[0][9:], k) for k, v in allFormats.items()])
+ validValues = sorted(reverseFormats.keys())
+ regex = "(?P<printFormat>A(HEX)?)(?P<printWid>\d+)"
+ isStringVar = re.compile(regex, re.IGNORECASE)
+ regex = "(?P<printFormat>[A-Z]+)(?P<printWid>\d+)\.?(?P<printDec>\d*)"
+ isAnyVar = re.compile(regex, re.IGNORECASE)
+ funcP = self.spssio.spssSetVarPrintFormat # print type
+ funcW = self.spssio.spssSetVarWriteFormat # write type
+ for varName, format_ in self.encode(formats).iteritems():
+ format_ = format_.upper()
+ gotString = isStringVar.match(format_)
+ gotAny = isAnyVar.match(format_)
+ msg = ("Unknown format_ %r for variable %r. " +
+ "Valid formats are: %s")
+ msg = msg % (", ".join(validValues), format_, varName)
+ if gotString:
+ printFormat = gotString.group("printFormat")
+ printFormat = reverseFormats.get(printFormat)
+ printDec = 0
+ printWid = int(gotString.group("printWid"))
+ elif gotAny:
+ printFormat = gotAny.group("printFormat")
+ printFormat = reverseFormats.get(printFormat)
+ printDec = gotAny.group("printDec")
+ printDec = int(printDec) if printDec else 0
+ printWid = int(gotAny.group("printWid"))
+ else:
+ raise ValueError(msg)
+
+ if printFormat is None:
+ raise ValueError(msg)
+
+ args = (c_int(self.fh), c_char_p(varName), c_int(printFormat),
+ c_int(printDec), c_int(printWid))
+ retcode1, retcode2 = funcP(*args), funcW(*args)
+ if retcodes.get(retcode1) == "SPSS_INVALID_PRFOR":
+ # invalid PRint FORmat
+ msg = "format_ for %r misspecified (%r)"
+ raise SPSSIOError(msg % (varName, format_), retcode1)
+ elif retcode1 > 0:
+ msg = "Error setting format_ %r for %r"
+ raise SPSSIOError(msg % (format_, varName), retcode1)
+
+ def _getMissingValue(self, varName):
+ """This is a helper function for the missingValues getter
+ method. The function returns the missing values of variable <varName>
+ as a a dictionary. The dictionary keys and items depend on the
+ particular definition, which may be discrete values and/or ranges.
+ Range definitions are only possible for numerical variables."""
+ if self.varTypes[varName] == 0:
+ func = self.spssio.spssGetVarNMissingValues
+ args = (c_double(), c_double(), c_double())
+ else:
+ func = self.spssio.spssGetVarCMissingValues
+ args = (create_string_buffer(9), create_string_buffer(9),
+ create_string_buffer(9))
+ missingFmt = c_int()
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(missingFmt), *map(byref, args))
+ if retcode > 0:
+ msg = "Error getting missing value for variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+
+ v1, v2, v3 = [v.value for v in args]
+ userMiss = dict([(v, k) for k, v in userMissingValues.iteritems()])
+ missingFmt = userMiss[missingFmt.value]
+ if missingFmt == "SPSS_NO_MISSVAL":
+ return {}
+ elif missingFmt == "SPSS_ONE_MISSVAL":
+ return {"values": [v1]}
+ elif missingFmt == "SPSS_TWO_MISSVAL":
+ return {"values": [v1, v2]}
+ elif missingFmt == "SPSS_THREE_MISSVAL":
+ return {"values": [v1, v2, v3]}
+ elif missingFmt == "SPSS_MISS_RANGE":
+ return {"lower": v1, "upper": v2}
+ elif missingFmt == "SPSS_MISS_RANGEANDVAL":
+ return {"lower": v1, "upper": v2, "value": v3}
+
+ def _setMissingValue(self, varName, **kwargs):
+ """This is a helper function for the missingValues setter
+ method. The function sets missing values for variable <varName>.
+ Valid keyword arguments are:
+ * to specify a RANGE: 'lower', 'upper', optionally with 'value'
+ * to specify DISCRETE VALUES: 'values', specified as a list no longer
+ than three items, or as None, or as a float/int/str
+ """
+ if kwargs == {}:
return 0
-
- def getInterfaceEncoding(self, spssio):
- """This function returns the current interface encoding.
- ('UTF-8' or 'CODEPAGE') and the specific current codepage (e.g. cp1252)"""
- try:
- swapped = dict(zip(self.icodes.values(), self.icodes.keys()))
- interfaceEncoding = swapped[spssio.spssGetInterfaceEncoding()]
- encoding = locale.getpreferredencoding()if interfaceEncoding == "CODEPAGE" else "UTF-8"
- return interfaceEncoding, encoding
- except AttributeError:
- print "NOTE. Function 'setInterfaceEncoding' not found. You are using a .dll from SPSS < v16"
- return "UTF-8", "UTF-8"
-
- def encodeStringValues(self, record, fileEncoding, encoding):
- """ This function encodes string values in a record in the encoding
- of the SPSS data file. """
- encodedRecord = []
- for value in record:
- if isinstance(value, str):
- try:
- value = value.decode(fileEncoding, "replace").encode(encoding)
- except UnicodeEncodeError:
- value = value.decode(fileEncoding, "replace").encode("UTF-8")
- encodedRecord.append(value)
- return encodedRecord
-
-
-######################################################################################
-
-class SavReader(object):
+ fargs = ["lower", "upper", "value", "values"]
+ if set(kwargs.keys()).difference(set(fargs)):
+ raise ValueError("Allowed keywords are: %s" % ", ".join(fargs))
+ varName = self.encode(varName)
+ varType = self.varTypes[varName]
+
+ # range of missing values, e.g. MISSING VALUES aNumVar (-9 THRU -1).
+ if varType == 0:
+ placeholder = 0.0
+ if "lower" in kwargs and "upper" in kwargs and "value" in kwargs:
+ missingFmt = "SPSS_MISS_RANGEANDVAL"
+ args = kwargs["lower"], kwargs["upper"], kwargs["value"]
+ elif "lower" in kwargs and "upper" in kwargs:
+ missingFmt = "SPSS_MISS_RANGE"
+ args = kwargs["lower"], kwargs["upper"], placeholder
+ else:
+ placeholder, args = "0", None
+
+ # up to three discrete missing values
+ if "values" in kwargs:
+ values = self.encode(kwargs.values()[0])
+ if isinstance(values, (float, int, str)):
+ values = [values]
+
+ # check if missing values strings values are not too long
+ strMissLabels = [len(v) for v in values if isinstance(v, str)]
+ if strMissLabels and max(strMissLabels) > 9:
+ raise ValueError("Missing value label > 9 bytes")
+
+ nvalues = len(values) if values is not None else values
+ if values is None or values == {}:
+ missingFmt = "SPSS_NO_MISSVAL"
+ args = placeholder, placeholder, placeholder
+ elif nvalues == 1:
+ missingFmt = "SPSS_ONE_MISSVAL"
+ args = values + [placeholder, placeholder]
+ elif nvalues == 2:
+ missingFmt = "SPSS_TWO_MISSVAL"
+ args = values + [placeholder]
+ elif nvalues == 3:
+ missingFmt = "SPSS_THREE_MISSVAL"
+ args = values
+ else:
+ msg = "You can specify up to three individual missing values"
+ raise ValueError(msg)
+
+ # numerical vars
+ if varType == 0 and args:
+ func = self.spssio.spssSetVarNMissingValues
+ func.argtypes = [c_int, c_char_p, c_int,
+ c_double, c_double, c_double]
+ args = map(float, args)
+ # string vars
+ else:
+ if args is None:
+ raise ValueError("Illegal keyword for character variable")
+ func = self.spssio.spssSetVarCMissingValues
+ func.argtypes = [c_int, c_char_p, c_int,
+ c_char_p, c_char_p, c_char_p]
+
+ retcode = func(self.fh, varName, userMissingValues[missingFmt], *args)
+ if retcode > 0:
+ msg = "Problem setting missing value of variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+
+ @property
+ @decode
+ def missingValues(self):
+ """Get/Set MISSING VALUES.
+ User missing values are values that will not be included in
+ calculations by SPSS. For example, 'don't know' might be coded as a
+ user missing value (a value of 999 is typically used, so when vairable
+ 'age' has values 5, 15, 999, the average age is 10). This is
+ different from 'system missing values', which are blank/null values.
+ Takes a dictionary of the following form:
+ {'someNumvar1': {'values': [999, -1, -2]}, # discrete values
+ 'someNumvar2': {'lower': -9, 'upper':-1}, # range "-9 THRU -1"
+ 'someNumvar3': {'lower': -9, 'upper':-1, 'value': 999},
+ 'someStrvar1': {'values': ['foo', 'bar', 'baz']},
+ 'someStrvar2': {'values': 'bletch'}} # shorthand """
+ missingValues = {}
+ for varName in self.varNames:
+ missingValues[varName] = self._getMissingValue(varName)
+ return missingValues
+
+ @missingValues.setter
+ def missingValues(self, missingValues):
+ if missingValues:
+ for varName, kwargs in missingValues.iteritems():
+ self._setMissingValue(varName, **kwargs)
+
+ # measurelevel, colwidth and alignment must all be set or not at all.
+ @property
+ @decode
+ def measureLevels(self):
+ """Get/Set VARIABLE LEVEL (measurement level).
+ Returns/Takes a dictionary of the form {varName: varMeasureLevel}.
+ Valid measurement levels are: "unknown", "nominal", "ordinal", "scale",
+ "ratio", "flag", "typeless". This is used in Spss procedures such as
+ CTABLES."""
+ func = self.spssio.spssGetVarMeasureLevel
+ levels = {0: "unknown", 1: "nominal", 2: "ordinal", 3: "scale",
+ 3: "ratio", 4: "flag", 5: "typeless"}
+ measureLevel = c_int()
+ varMeasureLevels = {}
+ for varName in self.varNames:
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(measureLevel))
+ varMeasureLevels[varName] = levels.get(measureLevel.value)
+ if retcode > 0:
+ msg = "Error getting variable measurement level: %r"
+ raise SPSSIOError(msg % varName, retcode)
+
+ return varMeasureLevels
+
+ @measureLevels.setter
+ def measureLevels(self, varMeasureLevels):
+ if not varMeasureLevels:
+ return
+ func = self.spssio.spssSetVarMeasureLevel
+ levels = {"unknown": 0, "nominal": 1, "ordinal": 2, "scale": 3,
+ "ratio": 3, "flag": 4, "typeless": 5}
+ for varName, level in self.encode(varMeasureLevels).iteritems():
+ if level.lower() not in levels:
+ msg = "Valid levels are %"
+ raise ValueError(msg % ", ".join(levels.keys()))
+ level = levels.get(level.lower())
+ retcode = func(c_int(self.fh), c_char_p(varName), c_int(level))
+ if retcode > 0:
+ msg = ("Error setting variable mesasurement level. " +
+ "Valid levels are: %s")
+ raise SPSSIOError(msg % ", ".join(levels.keys()), retcode)
+
+ @property
+ @decode
+ def columnWidths(self):
+ """Get/Set VARIABLE WIDTH (display width).
+ Returns/Takes a dictionary of the form {varName: <int>}. A value of
+ zero is special and means that the IBM SPSS Statistics Data Editor
+ is to set an appropriate width using its own algorithm. If used,
+ variable alignment, measurement level and column width all needs to
+ be set."""
+ func = self.spssio.spssGetVarColumnWidth
+ varColumnWidth = c_int()
+ varColumnWidths = {}
+ for varName in self.varNames:
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(varColumnWidth))
+ if retcode > 0:
+ msg = ("Error getting column width: %r" % varName)
+ raise SPSSIOError(msg, retcode)
+ varColumnWidths[varName] = varColumnWidth.value
+ return varColumnWidths
+
+ @columnWidths.setter
+ def columnWidths(self, varColumnWidths):
+ if not varColumnWidths:
+ return
+ func = self.spssio.spssSetVarColumnWidth
+ for varName, varColumnWidth in varColumnWidths.iteritems():
+ retcode = func(c_int(self.fh), c_char_p(varName),
+ c_int(varColumnWidth))
+ if retcode > 0:
+ msg = "Error setting variable colunm width"
+ raise SPSSIOError(msg, retcode)
+
+ def _setColWidth10(self):
+ """Set the variable display width of string values to at least 10
+ (it's annoying that SPSS displays e.g. a one-character variable in
+ very narrow columns). This also sets all measurement levels to
+ "unknown" and all variable alignments to "left". This function is
+ only called if column widths, measurement levels and variable
+ alignments are None."""
+ columnWidths = {}
+ for varName, varType in self.varTypes.iteritems():
+ # zero = appropriate width determined by spss
+ columnWidths[varName] = 10 if 0 < varType < 10 else 0
+ self.columnWidths = columnWidths
+ self.measureLevels = dict([(v, "unknown") for v in self.varNames])
+ self.alignments = dict([(v, "left") for v in self.varNames])
+
+ @property
+ @decode
+ def alignments(self):
+ """Get/Set VARIABLE ALIGNMENT.
+ Returns/Takes a dictionary of the form {varName: alignment}
+ Valid alignment values are: left, right, center. If used, variable
+ alignment, measurement level and column width all need to be set.
+ """
+ func = self.spssio.spssGetVarAlignment
+ alignments = {0: "left", 1: "right", 2: "center"}
+ alignment_ = c_int()
+ varAlignments = {}
+ for varName in self.varNames:
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(alignment_))
+ alignment = alignments[alignment_.value]
+ varAlignments[varName] = alignment
+ if retcode > 0:
+ msg = ("Error getting variable alignment: %r" % varName)
+ raise SPSSIOError(msg, retcode)
+ return varAlignments
+
+ @alignments.setter
+ def alignments(self, varAlignments):
+ if not varAlignments:
+ return
+ func = self.spssio.spssSetVarAlignment
+ alignments = {"left": 0, "right": 1, "center": 2}
+ for varName, varAlignment in varAlignments.iteritems():
+ if varAlignment.lower() not in alignments:
+ msg = "Valid alignments are %"
+ raise ValueError(msg % ", ".join(alignments.keys()))
+ alignment = alignments.get(varAlignment.lower())
+ retcode = func(c_int(self.fh), c_char_p(varName), c_int(alignment))
+ if retcode > 0:
+ msg = "Error setting variable alignment for variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+
+ @property
+ @decode
+ def varSets(self):
+ """Get/Set VARIABLE SET information.
+ Returns/Takes a dictionary with SETNAME as keys and a list of SPSS
+ variables as values. For example: {'SALARY': ['salbegin',
+ 'salary'], 'DEMOGR': ['gender', 'minority', 'educ']}"""
+ varSets = c_char_p()
+ func = self.spssio.spssGetVariableSets
+ retcode = func(c_int(self.fh), byref(varSets))
+ if retcode > 0:
+ msg = "Problem getting variable set information"
+ raise SPSSIOError(msg, retcode)
+ elif retcode == 0:
+ varSets_ = {}
+ for varSet in varSets.value.split("\n")[:-1]:
+ k, v = varSet.split("= ")
+ varSets_[k] = v.split()
+ self.freeMemoryItems["spssFreeVariableSets"] = (varSets,)
+ return varSets_
+ return {}
+
+ @varSets.setter
+ def varSets(self, varSets):
+ if not varSets:
+ return
+ varSets_ = []
+ for varName, varSet in varSets.iteritems():
+ varSets_.append("%s= %s" % (varName, " ".join(varSet)))
+ varSets_ = c_char_p("\n".join(varSets_))
+ retcode = self.spssio.spssSetVariableSets(c_int(self.fh), varSets_)
+ if retcode > 0:
+ msg = "Problem setting variable set information"
+ raise SPSSIOError(msg, retcode)
+
+ @property
+ @decode
+ def varRoles(self):
+ """Get/Set VARIABLE ROLES.
+ Returns/Takes a dictionary of the form {varName: varRole}, where
+ varRoles may be any of the following: 'both', 'frequency', 'input',
+ 'none', 'partition', 'record ID', 'split', 'target'"""
+ func = self.spssio.spssGetVarRole
+ roles = {0: "input", 1: "target", 2: "both", 3: "none", 4: "partition",
+ 5: "split", 6: "frequency", 7: "record ID"}
+ varRoles = {}
+ varRole_ = c_int()
+ for varName in self.varNames:
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName), byref(varRole_))
+ varRole = roles.get(varRole_.value)
+ varRoles[varName] = varRole
+ if retcode > 0:
+ msg = "Problem getting variable role for variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+ return varRoles
+
+ @varRoles.setter
+ def varRoles(self, varRoles):
+ if not varRoles:
+ return
+ roles = {"input": 0, "target": 1, "both": 2, "none": 3, "partition": 4,
+ "split": 5, "frequency": 6, "record ID": 7}
+ func = self.spssio.spssSetVarRole
+ for varName, varRole in varRoles.iteritems():
+ varRole = roles.get(varRole)
+ retcode = func(c_int(self.fh), c_char_p(varName), c_int(varRole))
+ if retcode > 0:
+ msg = "Problem setting variable role %r for variable %r"
+ raise SPSSIOError(msg % (varRole, varName), retcode)
+
+ @property
+ @decode
+ def varAttributes(self):
+ """Get/Set VARIABLE ATTRIBUTES.
+ Returns/Takes dictionary of the form:
+ {'var1': {'attr name x': 'attr value x','attr name y': 'attr value y'},
+ 'var2': {'attr name a': 'attr value a','attr name b': 'attr value b'}}
+ """
+ numVars = len(self.varNames)
+ attrNamesArr = (POINTER(c_char_p) * numVars)()
+ attrValuesArr = (POINTER(c_char_p) * numVars)()
+ nAttr, attributes = c_int(), {}
+ func = self.spssio.spssGetVarAttributes
+ for varName in self.varNames:
+ vName = self.vNames[varName]
+ retcode = func(c_int(self.fh), c_char_p(vName),
+ byref(attrNamesArr), byref(attrValuesArr),
+ byref(nAttr))
+ if retcode > 0:
+ msg = "Problem getting variable attributes for variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+ k, v, n = attrNamesArr[0], attrValuesArr[0], nAttr.value
+ attributes[varName] = dict([(k[i], v[i]) for i in xrange(n)])
+ funcStr = "spssFreeAttributes varAttributes"
+ self.freeMemoryItems[funcStr] = (attrNamesArr, attrValuesArr, nAttr)
+ return attributes
+
+ @varAttributes.setter
+ def varAttributes(self, varAttributes):
+ if not varAttributes:
+ return
+ func = self.spssio.spssSetVarAttributes
+ for varName in self.varNames:
+ attributes = varAttributes.get(varName)
+ if not attributes:
+ continue
+ nAttr = len(attributes)
+ attrNames = (c_char_p * nAttr)(*attributes.keys())
+ attrValues = (c_char_p * nAttr)(*attributes.values())
+ retcode = func(c_int(self.fh), c_char_p(varName),
+ pointer(attrNames), pointer(attrValues),
+ c_int(nAttr))
+ if retcode > 0:
+ msg = "Problem setting variable attributes for variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+
+ @property
+ @decode
+ def fileAttributes(self):
+ """Get/Set DATAFILE ATTRIBUTES.
+ Returns/Takes a dictionary of the form:
+ {'attrName[1]': 'attrValue1', 'revision[1]': '2010-10-09',
+ 'revision[2]': '2010-10-22', 'revision[3]': '2010-11-19'}
+ """
+ MAX_ARRAY_SIZE = 100
+ nAttr = c_int()
+ attrNamesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
+ attrValuesArr = (POINTER(c_char_p) * MAX_ARRAY_SIZE)()
+ func = self.spssio.spssGetFileAttributes
+ retcode = func(c_int(self.fh), byref(attrNamesArr),
+ byref(attrValuesArr), byref(nAttr))
+ if retcode > 0:
+ raise SPSSIOError("Problem getting file attributes", retcode)
+ k, v = attrNamesArr[0], attrValuesArr[0]
+ attributes = dict([(k[i], v[i]) for i in xrange(nAttr.value)])
+ funcStr = "spssFreeAttributes fileAttributes"
+ self.freeMemoryItems[funcStr] = (attrNamesArr, attrValuesArr, nAttr)
+ return attributes
+
+ @fileAttributes.setter
+ def fileAttributes(self, fileAttributes):
+ if not fileAttributes:
+ return
+ attributes, valueLens = {}, []
+ for name, values in fileAttributes.iteritems():
+ valueLens.append(len(values))
+ for value in values:
+ attributes[name] = value
+ #nAttr = len(fileAttributes)
+ nAttr = max(valueLens) # n elements per vector. But this may vary??
+ attrNames = (c_char_p * nAttr)(*attributes.keys())
+ attrValues = (c_char_p * nAttr)(*attributes.values())
+ func = self.spssio.spssSetFileAttributes
+ retcode = func(c_int(self.fh), pointer(attrNames),
+ pointer(attrValues), c_int(nAttr))
+ if retcode > 0:
+ raise SPSSIOError("Problem setting file attributes", retcode)
+
+ def _getMultRespDef(self, mrDef):
+ """Get 'normal' multiple response defintions.
+ This is a helper function for the multRespDefs getter function.
+ A multiple response definition <mrDef> in the string format returned
+ by the IO module is converted into a multiple response definition of
+ the form multRespSet = {<setName>: {"setType": <setType>, "label":
+ <lbl>, "varNames": <list_of_varNames>}}. SetType may be either 'D'
+ (multiple dichotomy sets) or 'C' (multiple category sets). If setType
+ is 'D', the multiple response definition also includes '"countedValue":
+ countedValue'"""
+ regex = "\$(?P<setName>\w+)=(?P<setType>[CD])\n?"
+ regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
+ m = re.search(regex + ".*", mrDef, re.I | re.U)
+ if not m:
+ return {}
+ setType = m.group("setType")
+ if setType == "C": # multiple category sets
+ regex += " (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?"
+ regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
+ matches = re.findall(regex, mrDef, re.I)
+ setName, setType, lblLen, lblVarNames = matches[0]
+ else: # multiple dichotomy sets
+ regex += ("(?P<valueLen>\d+) (?P<countedValue>\w+)" +
+ " (?P<lblLen>\d+) (?P<lblVarNames>.+) ?\n?")
+ regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
+ matches = re.findall(regex, mrDef, re.I | re.U)
+ setName, setType, valueLen = matches[0][:3]
+ countedValue, lblLen, lblVarNames = matches[0][3:]
+ lbl = lblVarNames[:int(lblLen)]
+ varNames = lblVarNames[int(lblLen):].split()
+ multRespSet = {setName: {"setType": setType, "label": lbl,
+ "varNames": varNames}}
+ if setType == "D":
+ multRespSet[setName]["countedValue"] = countedValue
+ return multRespSet
+
+ def _setMultRespDefs(self, multRespDefs):
+ """Set 'normal' multiple response defintions.
+ This is a helper function for the multRespDefs setter function.
+ It translates the multiple response definition, specified as a
+ dictionary, into a string that the IO module can use"""
+ mrespDefs = []
+ for setName, rest in multRespDefs.iteritems():
+ if rest["setType"] not in ("C", "D"):
+ return
+ rest["setName"] = setName
+ mrespDef = "$%(setName)s=%(setType)s" % rest
+ lblLen = len(mrespDef["label"])
+ rest["lblLen"] = lblLen
+ rest["varNames"] = " ".join(rest["varNames"])
+ tail = "%(varNames)s" if lblLen == 0 else "%(label)s %(varNames)s"
+ if mrespDef["setType"] == "C": # multiple category sets
+ template = " %%(lblLen)s %s " % tail
+ else: # multiple dichotomy sets
+ template = " %%(countedValue)s %%(lblLen)s %s " % tail
+ mrespDef += template % rest
+ mrespDefs.append(mrespDef)
+ mrespDefs = "\n".join(mrespDefs)
+ return mrespDefs
+
+ def _getMultRespDefsEx(self, mrDef):
+ """Get 'extended' multiple response defintions.
+ This is a helper function for the multRespDefs getter function."""
+ regex = ("\$(?P<setName>\w+)=(?P<setType>E) (?P<flag1>1)" +
+ "(?P<flag2>1)? (?P<valueLen>[0-9]+) (?P<countedValue>\w+) " +
+ "(?P<lblLen>[0-9]+) (?P<lblVarNames>[\w ]+)")
+ regex = unicode(regex, "utf-8") if self.ioUtf8 else regex
+ matches = re.findall(regex, mrDef, re.I | re.U)
+ setName, setType, flag1, flag2 = matches[0][:4]
+ valueLen, countedValue, lblLen, lblVarNames = matches[0][4:]
+ length = int(lblLen)
+ label, varNames = lblVarNames[:length], lblVarNames[length:].split()
+ return {setName: {"setType": setType, "firstVarIsLabel": bool(flag2),
+ "label": label, "countedValue": countedValue,
+ "varNames": varNames}}
+
+ def _setMultRespDefsEx(self, multRespDefs):
+ """Set 'extended' multiple response defintions.
+ This is a helper function for the multRespDefs setter function."""
+ mrDefs = []
+ for setName, rest in multRespDefs.iteritems():
+ if rest["setType"] != "E":
+ return {}
+ rest["setName"] = setName
+ v = int(rest["firstVarIsLabel"])
+ rest["firstVarIsLabel"] = v if v == 1 else ""
+ rest["valueLen"] = len(rest["countedValue"])
+ rest["lblLen"] = len(rest["label"])
+ rest["label"] = rest["label"]
+ rest["varNames"] = " ".join(rest["varNames"])
+ mrDef = "$%(setName)s=%(setType)s 1%(firstVarIsLabel)s "
+ mrDef += "%(valueLen)s %(countedValue)s %(lblLen)s %(label)s "
+ mrDef += "%(varNames)s"
+ mrDefs.append((mrDef % rest).replace(" ", " "))
+ return "\n".join(mrDefs)
+
+ @property
+ @decode
+ def multRespDefs(self):
+ """Get/Set MRSETS (multiple response) sets.
+ Returns/takes a dictionary of the form:
+ --multiple category sets: {setName: {"setType": "C", "label": lbl,
+ "varNames": [<list_of_varNames>]}}
+ --multiple dichotomy sets: {setName: {"setType": "D", "label": lbl,
+ "varNames": [<list_of_varNames>], "countedValue": countedValue}}
+ --extended multiple dichotomy sets: {setName: {"setType": "E",
+ "label": lbl, "varNames": [<list_of_varNames>], "countedValue":
+ countedValue, 'firstVarIsLabel': <bool>}}
+ For example:
+ categorical = {"setType": "C", "label": "labelC",
+ "varNames": ["salary", "educ"]}
+ dichotomous1 = {"setType": "D", "label": "labelD",
+ "varNames": ["salary", "educ"], "countedValue": "Yes"}
+ dichotomous2 = {"setType": "D", "label": "", "varNames":
+ ["salary", "educ", "jobcat"], "countedValue": "No"}
+ extended1 = {"setType": "E", "label": "", "varNames": ["mevar1",
+ "mevar2", "mevar3"], "countedValue": "1",
+ "firstVarIsLabel": True}
+ extended2 = {"setType": "E", "label":
+ "Enhanced set with user specified label", "varNames":
+ ["mevar4", "mevar5", "mevar6"], "countedValue":
+ "Yes", "firstVarIsLabel": False}
+ multRespDefs = {"testSetC": categorical, "testSetD1": dichotomous1,
+ "testSetD2": dichotomous2, "testSetEx1": extended1,
+ "testSetEx2": extended2}
+ """
+ if self.spssVersion[0] == 21:
+ print ("WARNING: getting multiple response definitions may not " +
+ "work with this IO version")
+ mrDefs = c_char_p()
+ func = self.spssio.spssGetMultRespDefs
+ retcode = func(c_int(self.fh), byref(mrDefs))
+ if retcode > 0:
+ msg = "Problem getting multiple response definitions"
+ raise SPSSIOError(msg, retcode)
+ multRespDefs = {}
+ if mrDefs.value:
+ for mrDef in mrDefs.value.split("\n"):
+ for setName, rest in self._getMultRespDef(mrDef).iteritems():
+ multRespDefs[setName] = rest
+ self.freeMemoryItems["spssFreeMultRespDefs Normal"] = (mrDefs,)
+
+ # I am not sure whether 'extended' MR definitions complement or replace
+ # 'normal' MR definitions. I assumed 'complement'.
+ mrDefsEx = c_char_p()
+ func = self.spssio.spssGetMultRespDefsEx
+ retcode = func(c_int(self.fh), byref(mrDefsEx))
+ if retcode > 0:
+ msg = "Problem getting extended multiple response definitions"
+ raise SPSSIOError(msg, retcode)
+ multRespDefsEx = {}
+ if mrDefsEx.value:
+ for mrDefEx in mrDefsEx.value.split("\n"):
+ for setName, rest in self._getMultRespDef(mrDefEx).iteritems():
+ multRespDefsEx[setName] = rest
+ self.freeMemoryItems["spssFreeMultRespDefs Extended"] = (mrDefsEx,)
+
+ multRespDefs.update(multRespDefsEx)
+ return multRespDefs
+
+ @multRespDefs.setter
+ def multRespDefs(self, multRespDefs):
+ if not multRespDefs:
+ return
+ normal = self._setMultRespDefs(multRespDefs)
+ extended = self._setMultRespDefsEx(multRespDefs)
+ if normal and extended:
+ combinedDefs = normal + " \n" + extended
+ elif normal and not extended:
+ combinedDefs = normal
+ elif extended and not normal:
+ combinedDefs = extended
+ func = self.spssio.spssSetMultRespDefs
+ retcode = func(c_int(self.fh), c_char_p(combinedDefs))
+ if retcode > 0:
+ msg = "Problem setting multiple response definitions"
+ raise SPSSIOError(msg, retcode)
+
+ @property
+ @decode
+ def caseWeightVar(self):
+ """Get/Set WEIGHT variable.
+ Takes a valid varName, and returns weight variable, if any, as a
+ string."""
+ varNameBuff = create_string_buffer(65)
+ func = self.spssio.spssGetCaseWeightVar
+ retcode = func(c_int(self.fh), byref(varNameBuff))
+ if retcode > 0:
+ msg = "Problem getting case weight variable name"
+ raise SPSSIOError(msg, retcode)
+ return varNameBuff.value
+
+ @caseWeightVar.setter
+ def caseWeightVar(self, varName):
+ if not varName:
+ return
+ func = self.spssio.spssSetCaseWeightVar
+ retcode = func(c_int(self.fh), c_char_p(varName))
+ if retcode > 0:
+ msg = "Problem setting case weight variable name %r" % varName
+ raise SPSSIOError(msg, retcode)
+
+ @property
+ @decode
+ def dateVariables(self): # entirely untested!
+ """Get/Set DATE information. This function reports the Forecasting
+ (Trends) date variable information, if any, in IBM SPSS Statistics
+ data files. Entirely untested and not implemented in reader/writer"""
+ nElements = c_int()
+ MAX_ARRAY_SIZE = 100 # or self.nCases?
+ dateInfoArr = (POINTER(c_long) * MAX_ARRAY_SIZE)()
+ func = self.spssio.spssGetDateVariables
+ retcode = func(c_int(self.fh), byref(nElements), byref(dateInfoArr))
+ if retcode > 0:
+ raise SPSSIOError("Error getting TRENDS information", retcode)
+ nElem = nElements.value
+ dateInfo = [dateInfoArr[0][i] for i in xrange(nElem)]
+ fixedDateInfo = dateInfo[:6]
+ otherDateInfo = [dateInfo[i: i + 3] for i in xrange(6, nElem, 3)]
+ self.freeMemoryItems["spssFreeDateVariables"] = (dateInfoArr,)
+ dateInfo = {"fixedDateInfo": fixedDateInfo,
+ "otherDateInfo": otherDateInfo}
+ return dateInfo
+
+ @dateVariables.setter
+ def dateVariables(self, dateInfo): # entirely untested!
+ dateInfo = dateInfo["fixedDateInfo"] + dateInfo["otherDateInfo"]
+ dateInfo = reduce(list.__add__, dateInfo) # flatten list
+ isAllFloats = all([isinstance(d, float) for d in dateInfo])
+ isSixPlusTriplets = (len(dateInfo) - 6) % 3 == 0
+ if not isAllFloats and isSixPlusTriplets:
+ msg = ("TRENDS date info must consist of 6 fixed elements" +
+ "+ <nCases> three-element groups of other date info " +
+ "(all floats)")
+ raise TypeError(msg)
+ func = self.spssio.spssSetDateVariables
+ dateInfoArr = (nElements * c_long)(*dateInfo)
+ retcode = func(c_int(self.fh), c_int(nElements), dateInfoArr)
+ if retcode > 0:
+ raise SPSSIOError("Error setting TRENDS information", retcode)
+
+ @property
+ @decode
+ def textInfo(self):
+ """Get/Set text information.
+ Takes a savFileName and returns a string of the form: "File %r built
+ using SavReaderWriter.py version %s (%s)". This is akin to, but
+ *not* equivalent to the SPSS syntax command DISPLAY DOCUMENTS"""
+ textInfo = create_string_buffer(256)
+ retcode = self.spssio.spssGetTextInfo(c_int(self.fh), byref(textInfo))
+ if retcode > 0:
+ raise SPSSIOError("Error getting textInfo", retcode)
+ return textInfo.value
+
+ @textInfo.setter
+ def textInfo(self, savFileName):
+ info = (os.path.basename(savFileName), __version__, time.asctime())
+ textInfo = "File '%s' built using SavReaderWriter.py version %s (%s)"
+ textInfo = textInfo % info
+ if self.ioUtf8 and isinstance(savFileName, unicode):
+ textInfo = textInfo.encode("utf-8")
+ func = self.spssio.spssSetTextInfo
+ retcode = func(c_int(self.fh), c_char_p(textInfo[:256]))
+ if retcode > 0:
+ raise SPSSIOError("Error setting textInfo", retcode)
+
+ @property
+ def fileCompression(self):
+ """Get/Set the file compression.
+ Returns/Takes a compression switch which may be any of the following:
+ 'uncompressed', 'standard', or 'zlib'. Zlib comression requires SPSS
+ v21 I/O files."""
+ compression = {0: "uncompressed", 1: "standard", 2: "zlib"}
+ compSwitch = c_int()
+ func = self.spssio.spssGetCompression
+ retcode = func(c_int(self.fh), byref(compSwitch))
+ if retcode > 0:
+ raise SPSSIOError("Error getting file compression", retcode)
+ return compression.get(compSwitch.value)
+
+ @fileCompression.setter
+ def fileCompression(self, compSwitch):
+ compression = {"uncompressed": 0, "standard": 1, "zlib": 2}
+ compSwitch = compression.get(compSwitch)
+ func = self.spssio.spssSetCompression
+ retcode = func(c_int(self.fh), c_int(compSwitch))
+ invalidSwitch = retcodes.get(retcode) == 'SPSS_INVALID_COMPSW'
+ if invalidSwitch and self.spssVersion[0] < 21:
+ msg = "Writing zcompressed files requires >=v21 SPSS I/O libraries"
+ raise ValueError(msg)
+ elif retcode > 0:
+ raise SPSSIOError("Error setting file compression", retcode)
+
+ @property
+ @decode
+ def fileLabel(self):
+ """Get/Set FILE LABEL (id string)
+ Takes a file label (basestring), and returns file label, if any, as
+ a string."""
+ idStr = create_string_buffer(65)
+ retcode = self.spssio.spssGetIdString(c_int(self.fh), byref(idStr))
+ if retcode > 0:
+ raise SPSSIOError("Error getting file label (id string)", retcode)
+ return idStr.value
+
+ @fileLabel.setter
+ def fileLabel(self, idStr):
+ if idStr is None:
+ idStr = "File created by user %r at %s"[:64] % \
+ (getpass.getuser(), time.asctime())
+ if self.ioUtf8 and isinstance(idStr, unicode):
+ idStr = idStr.encode("utf-8")
+ retcode = self.spssio.spssSetIdString(c_int(self.fh), c_char_p(idStr))
+ if retcode > 0:
+ raise SPSSIOError("Error setting file label (id string)", retcode)
+###############################################################################
+
+
+class SavHeaderReader(Header):
+ """
+ This class contains methods that read the data dictionary of an SPSS
+ data file. This yields the same information as the Spss command 'DISPLAY
+ DICTIONARY' NB: do not confuse an Spss dictionary with a Python
+ dictionary!
+
+ Typical use:
+ with SavHeaderReader(savFileName) as spssDict:
+ wholeDict = spssDict.dataDictionary()
+ print unicode(spssDict)
+ """
+
+ def __init__(self, savFileName, ioUtf8=False, ioLocale=None):
+ """ Constructor. Initializes all vars that can be recycled """
+ super(SavHeaderReader, self).__init__(savFileName, "rb", None,
+ ioUtf8, ioLocale)
+ self.fh = self.openSavFile()
+ self.varNames, self.varTypes = self.varNamesTypes
+ self.numVars = self.numberofVariables
+ self.nCases = self.numberofCases
+
+ def __str__(self):
+ """ This function returns a report of the SPSS data dictionary
+ (i.e., the header), in the encoding of the spss file"""
+ return unicode(self).encode(self.fileEncoding)
+
+ def __unicode__(self):
+ """ This function returns a report of the SPSS data dictionary
+ (i.e., the header)."""
+ report = ""
+ if self.textInfo:
+ report += self.textInfo + os.linesep
+ report += self.reportSpssDataDictionary(self.dataDictionary())
+ return report
+
+ def __enter__(self):
+ """ This function returns the DictionaryReader object itself so
+ its methods become available for use with context managers
+ ('with' statements)."""
+ return self
+
+ def __exit__(self, type, value, tb):
+ """ This function closes the spss data file and does some cleaning."""
+ if type is not None:
+ pass # Exception occurred
+ self.close()
+
+ def close(self):
+ """This function closes the spss data file and does some cleaning."""
+ if not segfaults:
+ self.freeMemory(self.freeMemoryItems)
+ self.closeSavFile(self.fh, mode="rb")
+ self.closeFile() # this in particular may cause errors in Linux
+
+ def dataDictionary(self):
+ """ This function returns all the dictionary items. It returns
+ a Python dictionary based on the Spss dictionary of the given
+ Spss file. This is equivalent to the Spss command 'DISPLAY
+ DICTIONARY'."""
+ items = ["varNames", "varTypes", "valueLabels", "varLabels",
+ "formats", "missingValues", "measureLevels",
+ "columnWidths", "alignments", "varSets", "varRoles",
+ "varAttributes", "fileAttributes", "fileLabel",
+ "multRespDefs", "caseWeightVar", "multRespDefs"]
+ if self.ioUtf8:
+ items = map(unicode, items)
+ dataDictionary = dict([(item, getattr(self, item)) for item in items])
+ return dataDictionary
+
+ def reportSpssDataDictionary(self, dataDict):
+ """ This function reports information from the Spss dictionary
+ of the active Spss dataset. The parameter 'dataDict' is the return
+ value of dataDictionary()"""
+ report = []
+ #import pprint
+ #pprint.pprint(dataDict)
+ for kwd, allValues in sorted(dataDict.items()):
+ report.append("#" + kwd.upper())
+ if hasattr(allValues, "items"):
+ for varName, values in allValues.iteritems():
+ if hasattr(values, "items"):
+ isList = kwd in ("missingValues", "multRespDefs")
+ for k, v in sorted(values.iteritems()):
+ if isList and isinstance(v, (list, tuple)):
+ vStr = map(unicode.lower, map(unicode, v))
+ report.append("%s: %s -- %s" %
+ (varName, k, ", ".join(vStr)))
+ else:
+ report.append("%s: %s -- %s" %
+ (varName, unicode(k).strip(), v))
+ else:
+ if isinstance(values, list):
+ entry = "%s -- %s" % (varName, ", ".join(values))
+ report.append(entry)
+ elif values != "":
+ report.append("%s -- %s" % (varName, values))
+ else:
+ if isinstance(allValues, basestring) and allValues:
+ allValues = [allValues]
+ for varName in allValues:
+ report.append(varName)
+ return os.linesep.join(report)
+###############################################################################
+
+
+class SavReader(Header):
""" Read Spss system files (.sav)
Parameters:
-savFileName: the file name of the spss data file
-returnHeader: Boolean that indicates whether the first record should
- be a list of variable names (default is True)
+ be a list of variable names (default = False)
-recodeSysmisTo: indicates to which value missing values should
- be recoded (default = ""),
+ be recoded (default = None),
-selectVars: indicates which variables in the file should be selected.
The variables should be specified as a list or a tuple of
valid variable names. If None is specified, all variables
in the file are used (default = None)
- -verbose: Boolean that indicates whether information about the spss data file
- (e.g., number of cases, variable names, file size) should be printed on
- the screen (default = True).
- -rawMode: Boolean that indicates whether values should get SPSS-style formatting,
- and whether date variables (if present) should be converted to ISO-dates. If True.
- the program does not format any values, which increases processing speed.
- (default = False)
- -interfaceEncoding: indicates the mode in which text communicated to or from the
- I/O Module will be. Valid values are 'UTF-8' or 'CODEPAGE' (default = 'CODEPAGE')
+ -idVar: indicates which variable in the file should be used for use as id
+ variable for the 'get' method (default = None)
+ -verbose: Boolean that indicates whether information about the spss data
+ file (e.g., number of cases, variable names, file size) should be
+ printed on the screen (default = False).
+ -rawMode: Boolean that indicates whether values should get SPSS-style
+ formatting, and whether date variables (if present) should be converted
+ to ISO-dates. If True, the program does not format any values, which
+ increases processing speed. (default = False)
+ -ioUtf8: indicates the mode in which text communicated to or from the
+ I/O Module will be. Valid values are True (UTF-8 mode aka Unicode mode)
+ and False (Codepage mode). Cf. SET UNICODE=ON/OFF (default = False)
+ -ioLocale: indicates the locale of the I/O module. Cf. SET LOCALE (default
+ = None, which corresponds to locale.getlocale()[0])
Typical use:
savFileName = "d:/someFile.sav"
- with SavReader(savFileName) as sav:
+ with SavReader(savFileName, returnHeader=True) as sav:
header = sav.next()
for line in sav:
process(line)
"""
- def __init__(self, savFileName, returnHeader=True, recodeSysmisTo="",
- verbose=True, selectVars=None, rawMode=False, interfaceEncoding="CODEPAGE"):
+ def __init__(self, savFileName, returnHeader=False, recodeSysmisTo=None,
+ verbose=False, selectVars=None, idVar=None, rawMode=False,
+ ioUtf8=False, ioLocale=None):
""" Constructor. Initializes all vars that can be recycled """
-
+ super(SavReader, self).__init__(savFileName, "rb", None,
+ ioUtf8, ioLocale)
self.savFileName = savFileName
self.returnHeader = returnHeader
self.recodeSysmisTo = recodeSysmisTo
self.verbose = verbose
self.selectVars = selectVars
+ self.idVar = idVar
self.rawMode = rawMode
- self.interfaceEncodingIn = interfaceEncoding
-
- self.gregorianEpoch = datetime.datetime(1582, 10, 14, 0, 0, 0)
-
- self.numVars = ctypes.c_int()
- self.numVarsPtr = ctypes.byref(self.numVars)
- self.nCases = ctypes.c_long()
- self.numofCasesPtr = ctypes.byref(self.nCases)
-
- self.attribNames = ctypes.c_char_p()
- self.attribText = ctypes.c_char_p()
- self.nAttributes = ctypes.c_int()
- self.attribNamesPtr = ctypes.byref(self.attribNames)
- self.attribTextPtr = ctypes.byref(self.attribText)
- self.nAttributesPtr = ctypes.byref(self.nAttributes)
-
- self.numValue = ctypes.c_double()
- self.numValuePtr = ctypes.byref(self.numValue)
- self.assumedCharWid = 200
- self.charValue = ctypes.create_string_buffer(self.assumedCharWid)
- self.charValuePtr = ctypes.byref(self.charValue)
- self.valueSize = ctypes.c_int(self.assumedCharWid)
-
- self.generic = Generic()
- self.spssio, self.fopen = self.generic.loadLibrary()
- self.sysmis = self.generic.getSystemSysmisVal(self.spssio)
- self.generic.setInterfaceEncoding(self.spssio, self.interfaceEncodingIn) or False
- self.retcode, self.fh = self.generic.loadSavFile(self.savFileName, self.spssio)
-
- self.SavDataDictionaryReader = SavDataDictionaryReader(self.savFileName)
-
- self.fileEncoding = self.getFileEncoding(self.fh)[1] or "UTF-8"
- self.fileCodePage = self.getFileCodePage(self.fh)[1] or locale.getpreferredencoding()
- #self.encoding = self.fileCodePage ## ???
-
- self.numVars_, self.nCases_, self.varNames, self.varTypes, self.printTypesFile, \
- self.printTypeLabels, self.varWids = self._readBasicSavFileInfo()
- self.structFmt = self.generic.conversionFormatCtoPy(self.varTypes, self.varNames)
-
- self.interfaceEncoding, self.encoding = self.generic.getInterfaceEncoding(self.spssio) or ("UTF-8", "UTF-8")
+
self.header = self.getHeader(self.selectVars)
+ self.bareformats, self.varWids = self._splitformats()
+ self.autoRawMode = self._isAutoRawMode()
+
+ self.ioUtf8_ = ioUtf8
+ self.sysmis_ = self.sysmis
+ self.numVars = self.numberofVariables
+ self.nCases = self.numberofCases
+
+ self.myStruct = self.getStruct(self.varTypes, self.varNames)
+ self.unpack_from = self.myStruct.unpack_from
+ self.seekNextCase = self.spssio.spssSeekNextCase
+ self.caseBuffer = self.getCaseBuffer()
+
+ if psycoOk:
+ self._items = psyco.proxy(self._items) # 3 x faster!
def __enter__(self):
- """ This function opens the spss data file."""
- return self.readSavFile(self.returnHeader, self.recodeSysmisTo,
- self.selectVars, self.rawMode, self.encoding)
+ """ This function opens the spss data file (context manager)."""
+ if self.verbose and self.ioUtf8_:
+ print unicode(self).replace(os.linesep, "\n")
+ elif self.verbose:
+ print str(self).replace(os.linesep, "\n")
+ return iter(self)
def __exit__(self, type, value, tb):
- """ This function closes the spss data file."""
+ """ This function closes the spss data file and does some cleaning."""
if type is not None:
- pass # Exception occurred
- self.spssio.spssCloseRead(self.fh)
+ pass # Exception occurred
+ self.close()
+
+ def close(self):
+ """This function closes the spss data file and does some cleaning."""
+ if not segfaults:
+ self.freeMemory(self.freeMemoryItems)
+ self.closeSavFile(self.fh, mode="rb")
+ self.closeFile()
del self.spssio
def __len__(self):
- """ This function reports the number of cases (rows) in the spss data file.
- For example: len(SavReader(savFileName)) """
- return self.nCases_
-
- def _readBasicSavFileInfo(self):
- """ This function reads and returns some basic information of the open
- spss data file. It returns the following variables:
- retcode: the return code (0 means OK)
- spssio: the spss i/o C module, opened with ctypes.windll.spssio32
- fh: the file handle
- numVars: the number of variables in the spss data file
- nCases: the number of cases (records) in the spss data file
- varNames: a list of the var names in the spss data file
- varTypes: a dictionary with var names as keys and var types as values
- printTypesFile: a dictionary with var names as keys and print types as values
- printTypeLabels: a dictionary with var names as keys and print type labels as values
- varWids: : a dictionary with var names as keys and var widths as values
- NOT FOR GENERAL USE; see getSavFileInfo
- """
-
- numVars = self.getNumberofVariables(self.fh, self.spssio)[1]
- nCases = self.getNumberofCases(self.fh, self.spssio)[1]
- varNames, varTypes_ = self.SavDataDictionaryReader.getVarInfo(self.fh, self.spssio)
-
- varTypes, printTypesFile, varWids, printDecs, \
- printWids = {}, {}, {}, {}, {}
- for i, varName in enumerate(varNames):
- varTypes[varName] = varTypes_[i]
- retcode, printType, printDec, printWid = \
- self.SavDataDictionaryReader.getVarPrintFormat(varName)
- printTypesFile[varName] = printType
- varWids[varName] = printWid
- printDecs[varName] = printDec
- printWids[varName] = printWid
-
- printTypeLabels = dict([(varName,
- printTypes[printType][0])
- for varName, printType in printTypesFile.iteritems()])
-
- fmts = dict([(varName, printTypeLabels[varName].split("_")[-1])
- for varName in varNames])
- if self.verbose:
- print self.getFileReport(self.savFileName, varNames, varTypes, fmts,
- printDecs, printWids, nCases)
-
- return numVars, nCases, varNames, varTypes, printTypesFile, printTypeLabels, varWids
+ """ This function reports the number of cases (rows) in the spss data
+ file. For example: len(SavReader(savFileName))"""
+ return self.nCases
+
+ def __cmp__(self, other):
+ """ This function implements behavior for all of the comparison
+ operators so comparisons can be made between SavReader instances,
+ or comparisons between SavReader instances and integers."""
+ if not isinstance(other, (SavReader, int)):
+ raise TypeError
+ other = other if isinstance(other, int) else len(other)
+ if len(self) < other:
+ return -1
+ elif len(self) == other:
+ return 0
+ else:
+ return 1
+
+ def __hash__(self):
+ """This function returns a hash value for the object to ensure it
+ is hashable."""
+ return id(self)
+
+ def __str__(self):
+ """This function returns a conscise file report of the spss data file
+ For example str(SavReader(savFileName))"""
+ return unicode(self).encode(self.fileEncoding)
+
+ def __unicode__(self):
+ """This function returns a conscise file report of the spss data file,
+ For example unicode(SavReader(savFileName))"""
+ self.fileReport = self.getFileReport(self.savFileName, self.varNames,
+ self.varTypes, self.formats,
+ self.nCases)
+ return self.fileReport
+
+ def _isAutoRawMode(self):
+ """Helper function for formatValues function. Determines whether
+ iterating over each individual value is really needed"""
+ hasDates = bool(set(self.bareformats.values()) & set(supportedDates))
+ hasNfmt = "N" in self.bareformats
+ hasRecodeSysmis = self.recodeSysmisTo is not None
+ items = [hasDates, hasNfmt, hasRecodeSysmis, self.ioUtf8_]
+ return False if any(items) else True
+
+ def formatValues(self, record):
+ """This function formats date fields to ISO dates (yyyy-mm-dd), plus
+ some other date/time formats. The SPSS N format is formatted to a
+ character value with leading zeroes. System missing values are recoded
+ to <recodeSysmisTo>. If rawMode==True, this function does nothing"""
+ if self.rawMode or self.autoRawMode:
+ return record # 6-7 times faster!
+
+ for i, value in enumerate(record):
+ varName = self.header[i]
+ varType = self.varTypes[varName]
+ bareformat_ = self.bareformats[varName]
+ varWid = self.varWids[varName]
+ if varType == 0:
+ # recode system missing values, if present and desired
+ if value > self.sysmis_:
+ pass
+ else:
+ record[i] = self.recodeSysmisTo
+ # format N-type values (=numerical with leading zeroes)
+ if bareformat_ == "N":
+ #record[i] = str(value).zfill(varWid)
+ record[i] = "%%0%dd" % varWid % value #15 x faster (zfill)
+ # convert SPSS dates to ISO dates
+ elif bareformat_ in supportedDates:
+ fmt = supportedDates[bareformat_]
+ args = (value, fmt, self.recodeSysmisTo)
+ record[i] = self.spss2strDate(*args)
+ elif varType > 0:
+ value = value[:varType]
+ if self.ioUtf8_:
+ record[i] = value.decode("utf-8")
+ else:
+ record[i] = value
+ return record
+
+ def _items(self, start=0, stop=None, step=1, returnHeader=False):
+ """ This is a helper function to implement the __getitem__ and
+ the __iter__ special methods. """
+
+ if returnHeader:
+ yield self.header
+
+ if all([start == 0, stop is None, step == 1]): # used as iterator
+ retcode = 0
+ else:
+ retcode = self.seekNextCase(c_int(self.fh), c_long(0)) # reset
+
+ if stop is None:
+ stop = self.nCases
+
+ selection = self.selectVars is not None
+ for case in xrange(start, stop, step):
+ if start:
+ # only call this when iterating over part of the records
+ retcode = self.seekNextCase(c_int(self.fh), c_long(case))
+ if retcode > 0:
+ raise SPSSIOError("Error seeking case %d" % case, retcode)
+ elif stop == self.nCases:
+ self.printPctProgress(case, self.nCases)
+
+ record = self.record
+
+ if selection:
+ record = self.selector(record)
+ record = [record] if len(record) == 1 else list(record)
+ record = self.formatValues(record)
+ yield record
+
+ def __iter__(self):
+ """This function allows the object to be used as an iterator"""
+ return self._items(0, None, 1, self.returnHeader)
+
+ def __getitem__(self, key):
+ """ This function reports the record of case number <key>.
+ For example: firstRecord = SavReader(savFileName)[0].
+ The <key> argument may also be a slice, for example:
+ firstfiveRecords = SavReader(savFileName)[:5].
+ You can also do stuff like (numpy required!):
+ savReader(savFileName)[1:5, 1]"""
+
+ is_slice = isinstance(key, slice)
+ is_array_slice = key is Ellipsis or isinstance(key, tuple)
+
+ if is_slice:
+ start, stop, step = key.indices(self.nCases)
+ elif is_array_slice:
+ return self._get_array_slice(key, self.nCases, len(self.header))
+ else:
+ key = operator.index(key)
+ start = key + self.nCases if key < 0 else key
+ if not 0 <= start < self.nCases:
+ raise IndexError("Index out of bounds")
+ stop = start + 1
+ step = 1
+
+ records = self._items(start, stop, step)
+ if is_slice:
+ return list(records)
+ return next(records)
+
+ def _get_array_slice(self, key, nRows, nCols):
+ """This is a helper function to implement array slicing with numpy"""
+
+ if not numpyOk:
+ raise ImportError("Array slicing requires the numpy library")
+
+ is_index = False
+ rstart = cstart = 0
+ try:
+ row, col = key
+ if row < 0:
+ row = nRows + row
+ if col < 0:
+ col = nCols + col
+
+ ## ... slices
+ if isinstance(row, slice) and col is Ellipsis:
+ # reader[1:2, ...]
+ rstart, rstop, rstep = row.indices(nRows)
+ cstart, cstop, cstep = 0, nRows, 1
+ elif row is Ellipsis and isinstance(col, slice):
+ # reader[..., 1:2]
+ rstart, rstop, rstep = 0, nRows, 1
+ cstart, cstop, cstep = col.indices(nCols)
+ elif isinstance(row, slice) and isinstance(col, slice):
+ # reader[1:2, 1:2]
+ rstart, rstop, rstep = row.indices(nRows)
+ cstart, cstop, cstep = col.indices(nCols)
+ elif row is Ellipsis and col is Ellipsis:
+ # reader[..., ...]
+ rstart, rstop, rstep = 0, nRows, 1
+ cstart, cstop, cstep = 0, nCols, 1
+
+ ## ... indexes
+ elif isinstance(row, int) and col is Ellipsis:
+ # reader[1, ...]
+ rstart, rstop, rstep = row, row + 1, 1
+ cstart, cstop, cstep = 0, nCols, 1
+ is_index = True
+ elif row is Ellipsis and isinstance(col, int):
+ # reader[..., 1]
+ rstart, rstop, rstep = 0, nRows, 1
+ cstart, cstop, cstep = col, col + 1, 1
+ is_index = True
+ elif isinstance(row, int) and isinstance(col, int):
+ # reader[1, 1]
+ rstart, rstop, rstep = row, row + 1, 1
+ cstart, cstop, cstep = col, col + 1, 1
+ is_index = True
+
+ # ... slice + index
+ elif isinstance(row, slice) and isinstance(col, int):
+ # reader[1:2, 1]
+ rstart, rstop, rstep = row.indices(nRows)
+ cstart, cstop, cstep = col, col + 1, 1
+ elif isinstance(row, int) and isinstance(col, slice):
+ # reader[1, 1:2]
+ rstart, rstop, rstep = row, row + 1, 1
+ cstart, cstop, cstep = col.indices(nCols)
+
+ try:
+ if not 0 <= rstart < nRows:
+ raise IndexError("Index out of bounds")
+ if not 0 <= cstart < nCols:
+ raise IndexError("Index out of bounds")
+
+ key = (Ellipsis, slice(cstart, cstop, cstep))
+
+ except UnboundLocalError:
+ msg = "The array index is either invalid, or not implemented"
+ raise TypeError(msg)
+
+ except TypeError:
+ # reader[...]
+ rstart, rstop, rstep = 0, nRows, 1
+ key = (Ellipsis, Ellipsis)
+
+ records = self._items(rstart, rstop, rstep)
+ result = numpy.array(list(records))[key].tolist()
+ if is_index:
+ return result[0]
+ return result
+
+ def head(self, n=5):
+ """ This convenience function returns the first <n> records. """
+ return self[:abs(n)]
+
+ def tail(self, n=5):
+ """ This convenience function returns the last <n> records. """
+ return self[-abs(n):]
+
+ def __contains__(self, item):
+ """ This function implements membership testing and returns True if
+ <idVar> contains <item>. Thus, it requires the 'idVar' parameter to
+ be set. For example: reader = SavReader(savFileName, idVar="ssn")
+ "987654321" in reader """
+ return bool(self.get(item))
+
+ def get(self, key, default=None, full=False):
+ """ This function returns the records for which <idVar> == <key>
+ if <key> in <savFileName>, else <default>. Thus, the function mimics
+ dict.get, but note that dict[key] is NOT implemented. NB: Even though
+ this uses a binary search, this is not very fast on large data (esp.
+ the first call, and with full=True)
+
+ Parameters:
+ -key: key for which the corresponding record should be returned
+ -default: value that should be returned if <key> is not found
+ (default: None)
+ -full: value that indicates whether *all* records for which
+ <idVar> == <key> should be returned (default: False)
+ For example: reader = SavReader(savFileName, idVar="ssn")
+ reader.get("987654321", "social security number not found!")"""
+
+ if not self.idVar in self.varNames:
+ msg = ("SavReader object must be instantiated with an existing " +
+ "variable as an idVar argument")
+ raise NameError(msg)
+
+ #two slightly modified functions from the bisect module
+ def bisect_right(a, x, lo=0, hi=None):
+ if hi is None:
+ hi = len(a)
+ while lo < hi:
+ mid = (lo + hi) // 2
+ if x < a[mid][0]:
+ hi = mid # a[mid][0], not a[mid]
+ else:
+ lo = mid + 1
+ return lo
+
+ def bisect_left(a, x, lo=0, hi=None):
+ if hi is None:
+ hi = len(a)
+ while lo < hi:
+ mid = (lo + hi) // 2
+ if a[mid][0] < x:
+ lo = mid + 1 # a[mid][0], not a[mid]
+ else:
+ hi = mid
+ return lo
+
+ idPos = self.varNames.index(self.idVar)
+ if not hasattr(self, "isSorted"):
+ self.isSorted = True
+ if self.varTypes[self.idVar] == 0:
+ if not isinstance(key, (int, float)):
+ return default
+ self.recordz = ((record[idPos], i) for i,
+ record in enumerate(iter(self)))
+ else:
+ if not isinstance(key, basestring):
+ return default
+ self.recordz = ((record[idPos].rstrip(), i) for i,
+ record in enumerate(iter(self)))
+ self.recordz = sorted(self.recordz)
+ insertLPos = bisect_left(self.recordz, key)
+ insertRPos = bisect_right(self.recordz, key)
+ if full:
+ result = [self[record[1]] for record in
+ self.recordz[insertLPos: insertRPos]]
+ else:
+ if insertLPos == insertRPos:
+ return default
+ result = self[self.recordz[insertLPos][1]]
+ if result:
+ return result
+ return default
def getSavFileInfo(self):
""" This function reads and returns some basic information of the open
- spss data file. Returns numVars, nCases, varNames, varTypes, printTypesFile,
- printTypeLabels, varWids. Suitable for use without context manager ('with' statement)
- See also _readBasicSavFileInfo method."""
- return self.numVars_, self.nCases_, self.varNames, self.varTypes, self.printTypesFile, \
- self.printTypeLabels, self.varWids
-
- def getNumberofVariables(self, fh, spssio):
- """ This function reports the number of variables present in a data file."""
- retcode = spssio.spssGetNumberofVariables(fh, self.numVarsPtr)
- return retcode, self.numVars.value
-
- def getNumberofCases(self, fh, spssio):
- """ This function reports the number of cases present in a data file"""
- retcode = spssio.spssGetNumberofCases(fh, self.numofCasesPtr)
- return retcode, self.nCases.value
-
- def formatValue(self, fh, spssio, variable, value, printTypeLabel,
- varWid, recodeSysmisTo):
- """ This function formats date fields to ISO dates (yyyy-mm-dd), plus
- some other date/time formats. The SPSS N format is formatted to a
- character value with leading zeroes."""
- supportedDates = {'SPSS_FMT_DATE': '%Y-%m-%d',
- 'SPSS_FMT_JDATE': '%Y-%m-%d',
- 'SPSS_FMT_EDATE': '%Y-%m-%d',
- 'SPSS_FMT_SDATE': '%Y-%m-%d',
- 'SPSS_FMT_DATE_TIME':'%Y-%m-%d %H:%M:%S',
- 'SPSS_FMT_WKDAY': '%A %H:%M:%S',
- 'SPSS_FMT_ADATE': '%Y-%m-%d',
- 'SPSS_FMT_WKDAY': '%A',
- 'SPSS_FMT_MONTH': '%B',
- 'SPSS_FMT_MOYR': '%B %Y',
- 'SPSS_FMT_WKYR': '%W WK %Y'}
- if printTypeLabel in supportedDates:
- fmt = supportedDates[printTypeLabel]
- return self.spss2strDate(value, fmt, recodeSysmisTo)
- elif printTypeLabel == 'SPSS_FMT_N':
- value = str(value).zfill(varWid)
- return value
- else:
- return value
-
+ spss data file."""
+ return (self.numVars, self.nCases, self.varNames, self.varTypes,
+ self.formats, self.varLabels, self.valueLabels)
+
+ def memoize(f):
+ """Memoization decorator
+ http://code.activestate.com/recipes/577219-minimalistic-memoization/"""
+ cache = {}
+ MAXCACHE = 10**4
+
+ def memf(*x):
+ if x not in cache:
+ cache[x] = f(*x)
+ if len(cache) > MAXCACHE:
+ cache.popitem()
+ return cache[x]
+ return memf
+
+ @memoize
def spss2strDate(self, spssDateValue, fmt, recodeSysmisTo):
- """ This function converts internal SPSS dates (number of seconds
+ """This function converts internal SPSS dates (number of seconds
since midnight, Oct 14, 1582 (the beginning of the Gregorian calendar))
- to a human-readable format """
+ to a human-readable format"""
try:
- theDate = self.gregorianEpoch + datetime.timedelta(seconds=spssDateValue)
+ if not hasattr(self, "gregorianEpoch"):
+ self.gregorianEpoch = datetime.datetime(1582, 10, 14, 0, 0, 0)
+ theDate = (self.gregorianEpoch +
+ datetime.timedelta(seconds=spssDateValue))
return datetime.datetime.strftime(theDate, fmt)
+ except OverflowError:
+ return recodeSysmisTo
except TypeError:
return recodeSysmisTo
except ValueError:
return recodeSysmisTo
- except OverflowError:
- return recodeSysmisTo
-
- def formatRecord(self, record, recodeSysmisTo):
- """ This function formats the values in a record according to the
- formats given in the SPSS file dictionary."""
- formattedRecord = []
- for rawValue, varName in zip(record, self.varNames):
- value = recodeSysmisTo if rawValue <= self.sysmis else rawValue
- if self.printTypeLabels[varName] != 'SPSS_FMT_F':
- value = self.formatValue(self.fh, self.spssio, varName, rawValue,
- self.printTypeLabels[varName],
- self.varWids[varName], recodeSysmisTo)
- formattedRecord.append(value)
- return formattedRecord
-
- def getFileEncoding(self, fh):
- """This function obtains the encoding applicable to a file.
- The encoding is returned as an IANA encoding name, such as
- ISO-8859-1. """
- self.pszEncoding = ctypes.create_string_buffer(20) # is 20 enough??
- self.pszEncodingPtr = ctypes.byref(self.pszEncoding)
- try:
- retcode = self.spssio.spssGetFileEncoding(self.fh, self.pszEncodingPtr)
- return retcode, self.pszEncoding.value
- except AttributeError:
- print "Function 'getFileEncoding' not found. You are using a .dll from SPSS < v16"
- return False, False
-
- def getFileCodePage(self, fh):
- """This function provides the Windows code page
- number of the encoding applicable to a file."""
- try:
- self.nCodePage = ctypes.c_int()
- self.nCodePagePtr = ctypes.byref(self.nCodePage)
- retcode = self.spssio.spssGetFileCodePage(self.fh, self.nCodePagePtr)
- return retcode, self.nCodePage.value
- except AttributeError:
- print "function 'spssGetFileCodePage' not found. You are using a .dll from SPSS < v16"
- return False, False
-
- def getFileReport(self, savFileName, varNames, varTypes, fmts, printDecs,
- printWids, nCases):
+
+ def getFileReport(self, savFileName, varNames, varTypes,
+ formats, nCases):
""" This function prints a report about basic file characteristics """
bytes = os.path.getsize(savFileName)
kb = float(bytes) / 2**10
mb = float(bytes) / 2**20
(fileSize, label) = (mb, "MB") if mb > 1 else (kb, "kB")
- line1 = [os.linesep + "*" * 70]
- line2 = ["*File '%s' (%5.2f %s) has %s columns (variables) and %s rows (%s values)" % \
- (savFileName, fileSize, label, len(varNames), nCases, len(varNames) * nCases)]
- line3 = ["*The file encoding is: %s (Code Page: %s)" % (self.fileEncoding, self.fileCodePage)]
- loc, cp = locale.getlocale()
- line4 = ["*Your computer's locale is: %s (Code page: %s)" % (loc, cp)]
- line5 = ["*The file contains the following variables:"]
- lines = []
+ systemString = self.systemString
+ spssVersion = ".".join(map(str, self.spssVersion))
+ lang, cp = locale.getlocale()
+ intEnc = "Utf-8/Unicode" if self.ioUtf8 else "Codepage (%s)" % cp
+ varlist = []
+ line = " %%0%sd. %%s (%%s - %%s)" % len(str(len(varNames) + 1))
for cnt, varName in enumerate(varNames):
- label = "string" if varTypes[varName] > 0 else "numerical"
- lines.append("%03d. %s (%s%d.%d - %s)" % (cnt+1, varName, fmts[varName], \
- printWids[varName], printDecs[varName], label))
- lineN = ["*" * 70]
- report = os.linesep.join(line1 + line2 + line3 + line4 + line5 + lines + lineN)
- return report
-
- def getCaseBuffer(self):
- """ This function returns a buffer and a pointer to that buffer. A whole
- case will be read into this buffer."""
- self.caseSize = ctypes.c_long()
- self.caseSizePtr = ctypes.byref(self.caseSize)
- self.retcode = self.spssio.spssGetCaseSize(self.fh, self.caseSizePtr)
- self.caseBuffer = ctypes.create_string_buffer(self.caseSize.value)
- self.caseBufferPtr = ctypes.byref(self.caseBuffer)
- return self.caseBuffer, self.caseBufferPtr
+ lbl = "string" if varTypes[varName] > 0 else "numerical"
+ format_ = formats[varName]
+ varlist.append(line % (cnt + 1, varName, format_, lbl))
+ info = {"savFileName": savFileName,
+ "fileSize": fileSize,
+ "label": label,
+ "nCases": nCases,
+ "nCols": len(varNames),
+ "nValues": nCases * len(varNames),
+ "spssVersion": "%s (%s)" % (systemString, spssVersion),
+ "ioLocale": self.ioLocale,
+ "ioUtf8": intEnc,
+ "fileEncoding": self.fileEncoding,
+ "fileCodePage": self.fileCodePage,
+ "isCompatible": "Yes" if self.isCompatibleEncoding() else "No",
+ "local_language": lang,
+ "local_encoding": cp,
+ "varlist": os.linesep.join(varlist),
+ "sep": os.linesep,
+ "asterisks": 70 * "*"}
+ report = ("%(asterisks)s%(sep)s" +
+ "*File %(savFileName)r (%(fileSize)3.2f %(label)s) has " +
+ "%(nCols)s columns (variables) and %(nCases)s rows " +
+ "(%(nValues)s values)%(sep)s" +
+ "*The file was created with SPSS version: %(spssVersion)s%" +
+ "(sep)s" +
+ "*The interface locale is: %(ioLocale)r%(sep)s" +
+ "*The interface mode is: %(ioUtf8)s%(sep)s" +
+ "*The file encoding is: %(fileEncoding)r (Code page: " +
+ "%(fileCodePage)s)%(sep)s" +
+ "*File encoding and the interface encoding are compatible:" +
+ " %(isCompatible)s%(sep)s" +
+ "*Your computer's locale is: %(local_language)r (Code " +
+ "page: %(local_encoding)s)%(sep)s" +
+ "*The file contains the following variables:%(sep)s" +
+ "%(varlist)s%(sep)s%(asterisks)s%(sep)s")
+ return report % info
def getHeader(self, selectVars):
+ """This function returns the variable names, or a selection thereof
+ (as specified as a list using the selectVars parameter), as a list."""
if selectVars is None:
header = self.varNames
elif isinstance(selectVars, (list, tuple)):
diff = set(selectVars).difference(set(self.varNames))
if diff:
- msg = "Variable names misspecified ('%s')" % ", ".join(diff)
- raise Exception, msg
- varPos = [self.varNames.index(v) for v in self.varNames if v in selectVars]
+ msg = "Variable names misspecified (%r)" % ", ".join(diff)
+ raise NameError(msg)
+ varPos = [varNames.index(v) for v in self.varNames
+ if v in selectVars]
self.selector = operator.itemgetter(*varPos)
header = self.selector(self.varNames)
header = [header] if not isinstance(header, tuple) else header
else:
- raise Exception, "Variable names list misspecified. " + \
- "Must be 'None' or a list or tuple of existing variables"
+ msg = ("Variable names list misspecified. Must be 'None' or a " +
+ "list or tuple of existing variables")
+ raise TypeError(msg)
return header
-
- def readSavFile(self, returnHeader, recodeSysmisTo, selectVars, rawMode, encoding):
- """ This is the main function of this class. It is a generator, which
- returns one record of the spss data file at a time. """
-
- debug = False
- if retcodes[self.retcode] == "SPSS_OK":
- if returnHeader:
- yield self.header
-
- # avoiding dots inside the loops
- # http://wiki.python.org/moin/PythonSpeed/PerformanceTips#Avoiding_dots...
- containsStringvars = max([varType for varName, varType in self.varTypes.items()
- if varName in self.header]) > 0
- self.caseBuffer, self.caseBufferPtr = self.getCaseBuffer()
- unpack = struct.unpack
- wholeCaseIn = self.spssio.spssWholeCaseIn
- print "Pct progress ...",
- for case in range(self.nCases_):
- retcode = wholeCaseIn(self.fh, self.caseBufferPtr)
- if retcodes[retcode] != 'SPSS_OK':
- print "WARNING: Record %s is faulty" % case+1
- continue
- record = unpack(self.structFmt, self.caseBuffer.raw)
- if selectVars is not None:
- record = self.selector(record)
- record = [record] if not isinstance(record, tuple) else record
- if containsStringvars:
- record = self.generic.encodeStringValues(record, self.fileEncoding, encoding)
- if not rawMode:
- record = self.formatRecord(record, self.recodeSysmisTo)
- if debug and (case+1) % 10 == 0:
- print "record", case+1, record
- pctProgress = (float(case) / self.nCases_) * 100
- if pctProgress % 5 == 0:
- print "%2.1f%%... " % pctProgress,
- yield record
- else:
- try:
- print "Error", retcodes[retcode]
- except KeyError:
- print "Unknown error code (%d)" % retcode
- finally:
- raise Exception, "You fail!"
-
-######################################################################################
-
-class SavWriter(object):
+###############################################################################
+
+
+class SavWriter(Header):
""" Write Spss system files (.sav)
Parameters:
- -savFileName: the file name of the spss data file
- -varNames: list or a tuple with the variable names of the spss data file.
- -varTypes: varTypes dictionary {varName: varType}, where varType == 0 means 'numeric',
- and varType > 0 means 'character' of that length
- -printTypesFile: optional printType dictionary {varName: spssFmt} (default: None)
- -overwrite: Boolean that indicates whether an existing Spss file should be overwritten
- (default: True)
-
-
+ * Formal
+ -savFileName: the file name of the spss data file. File names that end with
+ '.zsav' are compressed using the ZLIB (ZSAV) compression scheme
+ (requires v21 SPSS I/O files), while for file names that end with '.sav'
+ the 'old' compression scheme is used (it is not possible to generate
+ uncompressed files unless you modify the source code).
+ -varNames: list of the variable names in the order in which they appear in
+ in the spss data file.
+ -varTypes: varTypes dictionary {varName: varType}, where varType == 0 means
+ 'numeric', and varType > 0 means 'character' of that length (in bytes)
+
+ * Optional (the associated SPSS commands are given in CAPS)
+ -valueLabels: value label dictionary {varName: {value: label}} Cf. VALUE
+ LABELS (default: None)
+ -varLabels: variable label dictionary {varName: varLabel}. Cf. VARIABLE
+ LABEL (default: None)
+ -formats: format_ dictionary {varName: printFmt} Cf. FORMATS
+ (default: None)
+ -missingValues: missing values dictionary {varName: {missing value spec}}
+ (default: None). Cf. MISSING VALUES. For example:
+ {'someNumvar1': {'values': [999, -1, -2]}, # discrete values
+ 'someNumvar2': {'lower': -9, 'upper':-1}, # range, cf. -9 THRU -1
+ 'someNumvar3': {'lower': -9, 'upper':-1, 'value': 999},
+ 'someStrvar1': {'values': ['foo', 'bar', 'baz']},
+ 'someStrvar2': {'values': 'bletch'}}
+ ---The following three parameters must all three be set, if used---
+ -measureLevels: measurement level dictionary {varName: <level>}
+ Valid levels are: "unknown", "nominal", "ordinal", "scale",
+ "ratio", "flag", "typeless". Cf. VARIABLE LEVEL (default: None)
+ -columnWidths: column display width dictionary {varName: <int>}.
+ Cf. VARIABLE WIDTH. (default: None --> >= 10 [stringVars] or automatic
+ [numVars])
+ -alignments: alignment dictionary {varName: <left/center/right>}
+ Cf. VARIABLE ALIGNMENT (default: None --> left)
+ ---
+ -varSets: sets dictionary {setName: list_of_valid_varNames}.
+ Cf. SETSMR command. (default: None)
+ -varRoles: variable roles dictionary {setName: list_of_valid_varNames}.
+ Cf. VARIABLE ROLE (default: None)
+ -varAttributes: variable attributes dictionary {varName: {attribName:
+ attribValue} (default: None). For example: varAttributes = {'gender':
+ {'Binary': 'Yes'}, 'educ': {'DemographicVars': '1'}}. Cf. VARIABLE
+ ATTRIBUTES. (default: None)
+ -fileAttributes: file attributes dictionary {attribName: attribValue}
+ For example: {'RevisionDate[1]': '10/29/2004', 'RevisionDate[2]':
+ '10/21/2005'}. Square brackets indicate attribute arrays, which must
+ start with 1. Cf. FILE ATTRIBUTES. (default: None)
+ -fileLabel: file label string, which defaults to "File created by user
+ <username> at <datetime>" is file label is None. Cf. FILE LABEL
+ (default: None)
+ -multRespDefs: Multiple response sets definitions (dichotomy groups or
+ category groups) dictionary {setName: <set definition>}. In SPSS syntax,
+ 'setName' has a dollar prefix ('$someSet'). See also docstring of
+ multRespDefs method. Cf. MRSETS. (default: None)
+
+ -caseWeightVar: valid varName that is set as case weight (cf. WEIGHT BY)
+ -overwrite: Boolean that indicates whether an existing Spss file should be
+ overwritten (default: True)
+ -ioUtf8: indicates the mode in which text communicated to or from the
+ I/O Module will be. Valid values are True (UTF-8/unicode mode, cf. SET
+ UNICODE=ON) or False (Codepage mode, SET UNICODE=OFF) (default: False)
+ -ioLocale: indicates the locale of the I/O module, cf. SET LOCALE (default:
+ None, which is the same as locale.getlocale()[0])
+ -mode: indicates the mode in which <savFileName> should be opened. Possible
+ values are "wb" (write), "ab" (append), "cp" (copy: initialize header
+ using <refSavFileName> as a reference file, cf. APPLY DICTIONARY).
+ (default: "wb")
+ -refSavFileName: reference file that should be used to initialize the
+ header (aka the SPSS data dictionary) containing variable label, value
+ label, missing value, etc etc definitions. Only relevant in conjunction
+ with mode="cp". (default: None)
+
+ Typical use:
+ records = [['Test1', 1, 1], ['Test2', 2, 1]]
+ varNames = ['var1', 'v2', 'v3']
+ varTypes = {'var1': 5, 'v2': 0, 'v3': 0}
+ savFileName = "test.sav"
+ with SavWriter(savFileName, varNames, varTypes) as sav:
+ sav.writerows(records)
"""
-
- def __init__(self, savFileName, varNames, varTypes, printTypesFile=None,
- varLabels=None, valueLabels=None, overwrite=True,
- interfaceEncoding="CODEPAGE"):
+ def __init__(self, savFileName, varNames, varTypes, valueLabels=None,
+ varLabels=None, formats=None, missingValues=None,
+ measureLevels=None, columnWidths=None, alignments=None,
+ varSets=None, varRoles=None, varAttributes=None,
+ fileAttributes=None, fileLabel=None, multRespDefs=None,
+ caseWeightVar=None, overwrite=True, ioUtf8=False,
+ ioLocale=None, mode="wb", refSavFileName=None):
""" Constructor. Initializes all vars that can be recycled """
-
+ super(Header, self).__init__(savFileName, ioUtf8, ioLocale)
self.savFileName = savFileName
- self.varNames = varNames
- self.varTypes = varTypes
- self.printTypesFile = printTypesFile
+ self.varNames = self.encode(varNames)
+ self.varTypes = self.encode(varTypes)
self.overwrite = overwrite
- self.interfaceEncodingIn = interfaceEncoding
-
- self.generic = Generic()
- self.structFmt = self.generic.conversionFormatCtoPy(self.varTypes, self.varNames)
- self.spssio, self.fopen = self.generic.loadLibrary()
- self.sysmis = self.generic.getSystemSysmisVal(self.spssio)
- self.generic.setInterfaceEncoding(self.spssio, self.interfaceEncodingIn) or False
-
- self.fh = self.openWrite(self.savFileName, self.overwrite)[1]
- self.setVarNames(self.fh, self.varNames, self.varTypes)
-
- self.savDataDictionaryWriter = SavDataDictionaryWriter(self.fh, self.spssio, self.varNames, self.varTypes)
- self.savDataDictionaryWriter.setVarLabels(self.fh, self.spssio, varLabels)
- self.savDataDictionaryWriter.setValueLabels(self.fh, self.spssio, valueLabels)
- self.savDataDictionaryWriter.setVarPrintFormats(self.fh, self.varNames, self.varTypes, self.printTypesFile)
-
- self.commitHeader(self.fh)
- self.getCaseSize(self.fh)
- self.wholeCaseOut = self.spssio.spssWholeCaseOut
-
+ self.mode = mode
+ self.refSavFileName = refSavFileName
+
+ self.fh = super(Header, self).openSavFile(self.savFileName, self.mode,
+ self.refSavFileName)
+ self.myStruct = self.getStruct(self.varTypes, self.varNames, self.mode)
+ self.pack_into = self.myStruct.pack_into
+
+ self.sysmis_ = self.sysmis
+ self.ioUtf8_ = ioUtf8
+ self.pad_8_lookup = self._getPaddingLookupTable(self.varTypes)
+ #self._getVarHandles()
+
+ if self.mode == "wb":
+ self.openWrite(self.savFileName, self.overwrite)
+ self.varNamesTypes = self.varNames, self.varTypes
+ self.valueLabels = valueLabels
+ self.varLabels = varLabels
+ self.formats = formats
+ self.missingValues = missingValues
+ self.measureLevels = measureLevels
+ self.columnWidths = columnWidths
+ self.alignments = alignments
+ self.varSets = varSets
+ self.varRoles = varRoles
+ self.varAttributes = varAttributes
+ self.fileAttributes = fileAttributes
+ self.fileLabel = fileLabel
+ self.multRespDefs = multRespDefs
+ self.caseWeightVar = caseWeightVar
+ triplet = [measureLevels, columnWidths, alignments]
+ if all([item is None for item in triplet]):
+ self._setColWidth10()
+ self.textInfo = self.savFileName
+ self.commitHeader()
+ self.caseBuffer = self.getCaseBuffer()
+
def __enter__(self):
- """ This function returns the writer object itself so the writerow and
- the writerows methods become available for use with 'with' statements."""
+ """This function returns the writer object itself so the writerow and
+ writerows methods become available for use with 'with' statements"""
return self
def __exit__(self, type, value, tb):
""" This function closes the spss data file."""
if type is not None:
- pass # Exception occurred
- self.spssio.spssCloseWrite(self.fh)
- del self.spssio
+ pass # Exception occurred
+ self.closeSavFile(self.fh, mode="wb")
+ self.closeFile()
+
+ def _getVarHandles(self):
+ """This function returns a handle for a variable, which can then be
+ used to read or write (depending on how the file was opened) values
+ of the variable. If handle is associated with an output file, the
+ dictionary must be written with spssCommitHeader before variable
+ handles can be obtained via spssGetVarHandle. Helper function for
+ __setitem__. *Not currently used*"""
+ varHandles = {}
+ varHandle = c_double()
+ func = self.spssio.spssGetVarHandle
+ for varName in self.varNames:
+ retcode = func(c_int(self.fh), c_char_p(varName), byref(varHandle))
+ varHandles[varName] = varHandle.value
+ if retcode > 0:
+ msg = "Error getting variable handle for variable %r"
+ raise SPSSIOError(msg % varName, retcode)
+ return varHandles
+
+ def __setitem__(self, varName, value):
+ """This function sets the value of a variable for the current case.
+ The current case is not written out to the data file until
+ spssCommitCaseRecord is called. *Not currently used*, but it was just
+ begging to be implemented. ;-) For example:
+ self['someNumVar'] = 10
+ self['someStrVar'] = 'foo'"""
+ if not isinstance(value, (float, int, basestring)):
+ raise ValueError("Value %r has wrong type: %s" %
+ value, type(value))
+ varHandle = self.varHandles[varName]
+ if self.varTypes[varName] == 0:
+ funcN = self.spssio.spssSetValueNumeric
+ retcode = funcN(c_int(self.fh), c_double(varHandle),
+ c_double(value))
+ else:
+ funcC = self.spssio.spssSetValueChar
+ retcode = funcC(c_int(self.fh), c_double(varHandle),
+ c_char_p(value))
+ if retcode > 0:
+ isString = isinstance(value, basestring)
+ valType = "character" if isString else "numerical"
+ msg = "Error setting %s value %r for variable %r"
+ raise SPSSIOError(msg % (valType, value, varName), retcode)
def openWrite(self, savFileName, overwrite):
- """ This function opens a file in preparation for creating a new IBM SPSS Statistics data
- file and returns a handle that should be used for subsequent operations on the file."""
-
+ """ This function opens a file in preparation for creating a new IBM
+ SPSS Statistics data file"""
+ if os.path.exists(savFileName) and not os.access(savFileName, os.W_OK):
+ raise IOError("No write access for file %r" % savFileName)
+
if overwrite or not os.path.exists(savFileName):
- fh = self.fopen(self.savFileName, "wb")
- fhPtr = ctypes.byref(ctypes.c_int(fh))
- retcode = self.spssio.spssOpenWrite(ctypes.c_char_p(savFileName), fhPtr)
- compSwitch = ctypes.c_int(1) # compress file by default
- self.spssio.spssSetCompression(fh, compSwitch)
- #modified by RSS to get username from getpass.getuser() function
- idString = "File created by user '%s' at %s"[:64] % \
- (getpass.getuser(), time.asctime())
- self.spssio.spssSetIdString(fh, ctypes.create_string_buffer(idString))
- return retcode, fh
+ # always compress files, either zsav or standard.
+ if savFileName.lower().endswith(".zsav"):
+ self.fileCompression = "zlib" # only with v21 libraries!
+ else:
+ self.fileCompression = "standard"
elif not overwrite and os.path.exists(savFileName):
- raise Exception, "File '%s' already exists!" % self.savFileName
-
- def setVarNames(self, fh, varNames, varTypes):
- """ For each element of the dictionary VarTypes, this function creates a new variable
- named varName (the dictionary key), which will be either numeric or string based on
- varLength (the dictionary value). If the latter is zero, a numeric variable with a default
- format of F8.2 will be created; if it is greater than 0 and less than or equal to 32767,
- a string variable with length varLength will be created; any other value will be rejected as
- invalid."""
- for varName in varNames:
- varLength = varTypes[varName]
- retcode = self.spssio.spssValidateVarname(varName)
- msg = "invalid variable name (%s)[%s]" % (varName, retcodes[retcode])
- if retcode != 0:
- raise Exception, msg
- else:
- retcode2 = self.spssio.spssSetVarName(fh, varName, varLength)
- if retcode != 0:
- raise Exception, retcodes[retcode2]
-
- def commitHeader(self, fh):
- """This function writes the data dictionary to the data file associated with file handle 'fh'.
- Before any case data can be written, the dictionary must be committed; once the dictionary has
- been committed, no further changes can be made to it."""
- retcode = self.spssio.spssCommitHeader(fh)
- return retcode
-
- def getCaseSize(self, fh):
- """ This function reports the size of a raw case record for the file associated with handle.
- The case size is reported in bytes."""
- caseSize = ctypes.c_int()
- caseSizePtr = ctypes.byref(caseSize)
- retcode = self.spssio.spssGetCaseSize(fh, caseSizePtr)
- return retcode, caseSize.value
+ raise IOError("File %r already exists!" % savFileName)
+
+ def convertDate(self, day, month, year):
+ """This function converts a Gregorian date expressed as day-month-year
+ to the internal SPSS date format. The time portion of the date variable
+ is set to 0:00. To set the time portion if the date variable to another
+ value, use convertTime."""
+ d, m, y = c_int(day), c_int(month), c_int(year)
+ spssDate = c_double()
+ retcode = self.spssio.spssConvertDate(d, m, y, byref(spssDate))
+ if retcode > 0:
+ msg = "Problem converting date value '%s-%s-%s'" % (d, m, y)
+ raise SPSSIOError(msg, retcode)
+ return spssDate.value
+
+ def convertTime(self, day, hour, minute, second):
+ """This function converts a time given as day, hours, minutes, and
+ seconds to the internal SPSS time format."""
+ d, h, m, s, spssTime = (c_int(day), c_int(hour), c_int(minute),
+ c_double(float(second)), c_double())
+ retcode = self.spssio.spssConvertTime(d, h, m, s, byref(spssTime))
+ if retcode > 0:
+ msg = ("Problem converting time value '%s %s:%s:%s'" %
+ (day, hour, minute, second))
+ raise SPSSIOError(msg, retcode)
+ return spssTime.value
+
+ def spssDateTime(self, datetimeStr="2001-12-08", strptimeFmt="%Y-%m-%d"):
+ """ This function converts a date/time string into an SPSS date,
+ using a strptime format."""
+ dt = time.strptime(datetimeStr, strptimeFmt)
+ day, month, year = dt.tm_mday, dt.tm_mon, dt.tm_year
+ hour, minute, second = dt.tm_hour, dt.tm_min, dt.tm_sec
+ return (self.convertDate(day, month, year) +
+ self.convertTime(0, hour, minute, second))
+
+ def commitHeader(self):
+ """This function writes the data dictionary to the data file associated
+ with file handle 'fh'. Before any case data can be written, the
+ dictionary must be committed; once the dictionary has been committed,
+ no further changes can be made to it."""
+ retcode = self.spssio.spssCommitHeader(c_int(self.fh))
+ if retcode > 0:
+ raise SPSSIOError("Problem committing header", retcode)
+
+ def _getPaddingLookupTable(self, varTypes):
+ """Helper function that returns a lookup table that maps string lengths
+ to string lengths to the nearest ceiled multiple of 8. For example:
+ {1:%-8s, 7:%-8s, 9: %-16s, 24: %-24s}. Purpose: Get rid of trailing
+ null bytes"""
+ strLenghts = varTypes.values()
+ strRange = xrange(min(strLenghts), max(strLenghts) + 1)
+ return dict([(i, "%%-%ds" % (-8 * (i // -8))) for i in strRange])
def writerow(self, record):
""" This function writes one record, which is a Python list."""
- convertedRecord = []
- for value, varName in zip(record, self.varNames):
- charlen = self.varTypes[varName]
- if charlen > 0:
- value = value.ljust(charlen)
+ float_ = float
+ for i, value in enumerate(record):
+ varName = self.varNames[i]
+ varType = self.varTypes[varName]
+ if varType == 0:
+ try:
+ value = float_(value)
+ except ValueError:
+ value = self.sysmis_
else:
- try:
- value = float(value)
- except ValueError:
- value = self.sysmis
- convertedRecord.append(value)
- convertedRecord = self.generic.encodeStringValues(convertedRecord, "UTF-8", "UTF-8")
- retcode = self.wholeCaseOut(self.fh, struct.pack(self.structFmt, *convertedRecord))
- return retcode
+ # Get rid of trailing null bytes --> 7 x faster than 'ljust'
+ value = self.pad_8_lookup[varType] % value
+ if self.ioUtf8_:
+ if isinstance(value, unicode):
+ value = value.encode("utf-8")
+ record[i] = value
+ self.record = record
def writerows(self, records):
""" This function writes all records."""
- for record in records:
- print self.varNames
- retcode = self.writerow(record)
- if retcode < 0:
- print "Warning: %s" % retcodes[retcode]
- elif retcode > 0:
- raise Exception, "Error writing records. File in use? ('%s')" % \
- retcodes[retcode]
-
-
-##################################################################################
-
-class SavDataDictionaryReader(object):
- """ This class contains methods that read the data dictionary of an SPSS data file.
- This yields the same information as the Spss command 'DISPLAY DICTIONARY' (although
- not all dictionary items are implemented. NB: do not confuse an Spss dictionary with
- a Python dictionary!"""
-
- def __init__(self, savFileName):
- """ Constructor. Initializes all vars that can be recycled """
-
- self.savFileName = savFileName
- self.numVars = ctypes.c_int()
- self.numVarsPtr = ctypes.byref(self.numVars)
-
- self.printType = ctypes.c_int()
- self.printDec = ctypes.c_int()
- self.printWid = ctypes.c_int()
- self.printTypePtr = ctypes.byref(self.printType)
- self.printDecPtr = ctypes.byref(self.printDec)
- self.printWidPtr = ctypes.byref(self.printWid)
-
- self.generic = Generic()
- self.spssio, self.fopen = self.generic.loadLibrary()
- self.retcode, self.fh = self.generic.loadSavFile(self.savFileName, self.spssio)
-
- self.varNames, self.varTypes = self.getVarInfo(self.fh, self.spssio)
- self.varNamesTypes = dict([(varName, varType) for varName, varType in zip(self.varNames, self.varTypes)])
-
- def __enter__(self):
- """ This function returns the DictionaryReader object itself so its methods
- become available for use with context managers ('with' statements)."""
- return self
-
- def __exit__(self, type, value, tb):
- """ This function closes the spss data file and does some cleaning."""
- if type is not None:
- pass # Exception occurred
- self.spssio.spssCloseRead(self.fh)
- del self.spssio
-
- def getVarNameAndType(self, fh, spssio, iVar):
- """ Get variable name and type. The variable type code is an integer
- in the range 0-32767, 0 indicating a numeric variable and a positive
- value indicating a string variable of that size."""
- varNameBuff = ctypes.create_string_buffer(65)
- varNamePtr = ctypes.byref(varNameBuff)
- varType = ctypes.c_int()
- varTypePtr = ctypes.byref(varType)
- retcode = spssio.spssGetVarInfo(fh, iVar, varNamePtr, varTypePtr)
- return varNameBuff.value, varType.value
-
- def getVarInfo(self, fh, spssio):
- """ This function gets the name and type of one of the variables
- present in a data file."""
- spssio.spssGetNumberofVariables(fh, self.numVarsPtr)
- varNames, varTypes = [], []
- for iVar in range(self.numVars.value):
- varName, varType = self.getVarNameAndType(fh, spssio, iVar)
- varNames.append(varName)
- varTypes.append(varType)
- return varNames, varTypes
-
- def getVarLabel(self, varName):
- """ This function reports the Spss variable label, as defined by the
- Spss command "VARIABLE LABEL varName 'varLabel'." """
- MAX_SHORT_VARLABEL_LEN = 120 # fixed
- MAX_LONG_VAR_LEN = 1000 # allow variable labels of up to 1000 bytes
- varLabelBuff = ctypes.create_string_buffer(MAX_SHORT_VARLABEL_LEN + 1)
- varLabelPtr = ctypes.byref(varLabelBuff)
- retcode = self.spssio.spssGetVarLabel(self.fh, varName, varLabelPtr)
- varLabel = varLabelBuff.value
- if varLabel and len(varLabel) > MAX_SHORT_VARLABEL_LEN:
- lenBuff = MAX_LONG_VAR_LEN
- varLabelBuff = ctypes.create_string_buffer(lenBuff)
- varLabelBuffPtr = ctypes.byref(varLabelBuff)
- lenLabelPtr = ctypes.byref(ctypes.c_int())
- retcode = self.spssio.spssGetVarLabelLong (self.fh, varName, varLabelBuffPtr, lenBuff, lenLabelPtr)
- varLabel = varLabelBuff.value
- varLabel = unicode(varLabel, "utf-8", errors="ignore")
- return retcode, varName, varLabel
-
- def getValueLabels(self, varName):
- """This function gets the set of labeled values and associated labels for a short string
- variable or a numeric variable"""
-
- try:
- isNumVar = self.varNamesTypes[varName] == 0
- except KeyError:
- msg = "Variable '%s' does not exist!" % varName
- raise Exception, msg
-
- MAX_ARRAY_SIZE = 1000 # assume never more than 1000 labels
- numLabels = ctypes.c_int()
- numLabelsPtr = ctypes.byref(numLabels)
- if isNumVar:
- # Pointer to array of pointers double values
- valuesPtr = ctypes.pointer((ctypes.POINTER(ctypes.c_double) * MAX_ARRAY_SIZE)())
- # Pointer to array of pointers to labels
- labelsPtr = ctypes.pointer((ctypes.POINTER(ctypes.c_char_p) * MAX_ARRAY_SIZE)())
- retcode = self.spssio.spssGetVarNValueLabels(self.fh, varName, valuesPtr, labelsPtr, numLabelsPtr)
- else:
- valuesPtr = ctypes.pointer((ctypes.POINTER(ctypes.c_char_p) * MAX_ARRAY_SIZE)())
- labelsPtr = ctypes.pointer((ctypes.POINTER(ctypes.c_char_p) * MAX_ARRAY_SIZE)())
- retcode = self.spssio.spssGetVarCValueLabels(self.fh, varName, valuesPtr, labelsPtr, numLabelsPtr)
- valueLabels = {}
- for i in range(numLabels.value):
- value = valuesPtr.contents[0][i]
- label = labelsPtr.contents[0][i]
- if isinstance(value, str):
- value = unicode(value, "utf-8")
- label = unicode(label, "utf-8")
- valueLabels[value] = label
-
- # clean up (dunno yet why this crashes under Linux)
- # also crashes under Win 7. Outcoment if you dare ;-)
- #if sys.platform.lower().startswith("win"):
- # self.spssio.spssFreeVarNValueLabels(valuesPtr, labelsPtr, numLabels)
- # self.spssio.spssFreeVarCValueLabels(valuesPtr, labelsPtr, numLabels)
-
- return retcode, valueLabels
-
- def getVarPrintFormat(self, varName):
- """ This function reports the print format of a variable. Format
- type, number of decimal places, and field width are returned. """
- self.varName = ctypes.c_char_p(varName)
- retcode = self.spssio.spssGetVarPrintFormat(self.fh,
- self.varName,
- self.printTypePtr,
- self.printDecPtr,
- self.printWidPtr)
- return retcode, self.printType.value, self.printDec.value, \
- self.printWid.value
-
- def dataDictionary(self):
- """ This function returns all the dictionary items. It returns
- a Python dictionary based on the Spss dictionary of the given
- Spss file. This is equivalent to the Spss command 'DISPLAY
- DICTIONARY'. Currently only valueLabels, variableLabels, and
- printFormats are implemented."""
- dataDictionary = {}
- for varName in self.varNames:
- valueLabels = {varName: self.getValueLabels(varName)[1]}
- variableLabels = {varName: self.getVarLabel(varName)[-1]}
- printType, printDec, printWid = self.getVarPrintFormat(varName)[1:]
- fmt = printTypes[printType][0].split("_")[-1] + str(printWid) + "." + str(printDec)
- printFormats = {varName: fmt}
- try:
- dataDictionary["valueLabels"].append(valueLabels)
- dataDictionary["variableLabels"].append(variableLabels)
- dataDictionary["printFormats"].append(printFormats)
- except KeyError:
- dataDictionary["valueLabels"] = [valueLabels]
- dataDictionary["variableLabels"] = [variableLabels]
- dataDictionary["printFormats"] = [printFormats]
- return dataDictionary
-
- def reportSpssDataDictionary(self, dataDict):
- """ This function reports (prints) information from the dictionary
- of the active Spss dataset. The parameter 'dataDict' is the return
- value of dataDictionary()"""
- for kwd, items in dataDict.items():
- for item in dataDict[kwd]:
- for n, (varName, v) in enumerate(item.items()):
- if n == 0:
- print "\n* %s of variable '%s'" % (kwd.upper(), varName.upper())
- try:
- for value, label in v.items():
- print "%s -- %s" % (value, label)
- except AttributeError:
- print "%s -- %s" % (varName, v)
-
-##################################################################################
-
-class SavDataDictionaryWriter(object):
- """ This class allows some of the Spss data dictionary items to be set.
- None of the methods here should be called directly. All of them are used inside
- the SavWriter class"""
-
- def __init__(self, fh, spssio, varNames, varTypes):
- """ Constructor. Initializes all vars that can be recycled """
- self.varNames = varNames
- self.varTypes = varTypes.keys()
- self.varNamesTypes = varTypes
- self.generic = Generic()
- self.spssio, self.fopen = self.generic.loadLibrary()
-
- def setVarLabels(self, fh, spssio, varLabels=None):
- """ This function sets the label of a variable."""
- if varLabels:
- for varName, varLabel in varLabels.iteritems():
- retcode = spssio.spssSetVarLabel(fh, ctypes.c_char_p(varName), ctypes.c_char_p(varLabel))
- if retcode != 0:
- print "NOTE. Problem with setting variable label '%s' of variable '%s' (%s)" \
- % (varLabel, varName, retcodes[retcode])
-
- def setValueLabels(self, fh, spssio, allValueLabels):
- """ This function changes or adds the value labels of the
- values of a variable. The allValueLabels should be specified as a
- Python dictionary {varName: {value: valueLabel} or as None"""
- if allValueLabels:
- for varName, valueLabels in allValueLabels.iteritems():
- if valueLabels:
- isNumeric = self.varNamesTypes[varName] == 0
- for value, valueLabel in valueLabels.iteritems():
- if isNumeric:
- retcode = spssio.spssSetVarNValueLabel(fh, ctypes.c_char_p(varName),\
- ctypes.c_double(value), \
- ctypes.c_char_p(valueLabel))
- else:
- retcode = spssio.spssSetVarCValueLabel(fh, ctypes.c_char_p(varName),\
- ctypes.c_char_p(value), \
- ctypes.c_char_p(valueLabel))
- if retcode != 0:
- print "NOTE. Problem with setting value labels of variable %s ('%s')" % \
- (varName, retcodes[retcode])
-
- def setVarPrintFormats(self, fh, varNames, varTypes, printTypesFile):
- """ This function sets the print format of the variables."""
- # needs some refactoring ;-)
- setPrintFmt = self.spssio.spssSetVarPrintFormat
- if printTypesFile is not None:
- if isinstance(printTypesFile, dict):
- regex = re.compile("([A-Z]+)(\d+)\.*(\d*)")
- for varName in varNames:
- varType = varTypes[varName]
- thePrintType = printTypesFile[varName]
- m = re.search(regex, printTypesFile[varName])
- legalValues = [value[0][9:] for value in printTypes.values()]
- theValue = re.sub("[0-9.]+", "", thePrintType)
- if varType == 0 and m and theValue in legalValues:
- printType = reversePrintTypes[m.group(1)]
- printDec = 0 if not m.group(3) else int(m.group(3))
- printWid = int(m.group(2))
- retcode = setPrintFmt(fh, ctypes.create_string_buffer(varName),
- ctypes.c_int(printType),
- ctypes.c_int(printDec),
- ctypes.c_int(printWid))
- if retcodes[retcode] == "SPSS_INVALID_PRFOR": # = invalid PRint FORmat
- if m and theValue not in legalValues:
- msg = "Unknown printType format ('%s'), legal prefixes are: %s" % \
- (theValue, ", ".join(legalValues))
- elif m and printDec >= printWid:
- msg = "PrintType error: variable width greater or equal " + \
- "than number of decimals (variable '%s' has impossible " % varName + \
- "format '%s%s.%s')" % (theValue, printWid, printDec)
- elif not m:
- msg = "PrintType misspecified ('%s')" % thePrintType
- raise Exception, msg
- else:
- msg = """PrintTypes should be specified as a dictionary {'varname': printType}
- For example: {'v1': 'F2.2', 'v2': 'A30'}"""
- raise Exception, msg
- else:
- print "NOTE. No printTypes specified, using defaults"
-
-
-######################################################################################
+ nCases = len(records)
+ for case, record in enumerate(records):
+ self.writerow(record)
+ self.printPctProgress(case, nCases)
+
+###############################################################################
if __name__ == "__main__":
- import contextlib, csv
-
- #savFileName = r"C:\Program Files\IBM\SPSS\Statistics\19\Samples\English\Employee data.sav"
- savFileName = "c:/program files/spss/employee data.sav"
- tempdir = tempfile.gettempdir() # changed to std function for Mac compatibilityos.environ["TEMP"]
-
- ## ---------------------------------
+ import contextlib
+ import csv
+ import collections
+ import pprint
+ import cProfile
+ import pstats
+
+ #savFileName = r"C:\Program Files\IBM\SPSS\Statistics\20\Samples\English\Employee data.sav"
+
+ ## ---------------------------------
+ ## SAV DATA DICTIONARY READER
+ ## ---------------------------------
+
+ ## ----- Typical use
+ with SavHeaderReader("employee data.sav") as spssDict:
+ print str(spssDict) # prints a report of the header information
+ wholeDict = spssDict.dataDictionary()
+
+ ## ----- Copy header info from one file to another
+ savFileName = "employee data.sav"
+ with SavHeaderReader(savFileName) as spssDict:
+ wholeDict = spssDict.dataDictionary()
+ reader = SavReader(savFileName)
+ writer = SavWriter('test_wholeDict.sav', **wholeDict)
+ with contextlib.nested(reader, writer) as (r, w):
+ for line in r:
+ w.writerow(line)
+
+ ## ---------------------------------
## SAV READER
## ---------------------------------
- ## ----- Get some basic file info
- numVars, nCases, varNames, varTypes, printTypesFile, printTypeLabels, varWids = \
- SavReader(savFileName).getSavFileInfo()
-
- print "The file contains %d records" % ( len(SavReader(savFileName)) )
-
- ## ----- Typical use
- with SavReader(savFileName, selectVars=['id'], recodeSysmisTo=999) as sav:
+ ## ----- Typical use
+ savFileName = "employee data.sav"
+ with SavReader(savFileName, returnHeader=True) as sav:
header = sav.next()
for line in sav:
- pass # do stuff
-
- ## ----- Convert file to .csv
- csvFileName = os.path.join(tempdir, "test_out.csv")
- with contextlib.nested(SavReader(savFileName, selectVars=None, verbose=True,
- rawMode=False, interfaceEncoding="UTF-8"),
- open(csvFileName, "wb")) as (sav, f):
+ pass # do stuff
+
+ ## ----- Explore the data file.
+ ## ... Get some basic file info
+ reader = SavReader(savFileName, idVar="id")
+ (numVars, nCases, varNames, varTypes,
+ formats, varLabels, valueLabels) = reader.getSavFileInfo()
+ print "The file contains %d records" % len(reader)
+ print str(reader) # prints a file report
+
+ ## ... Use indexing, slicing, or array slicing to access records
+ print "The first six records look like this\n", reader[:6]
+ print "The first record looks like this\n", reader[0]
+ print "The last four records look like this\n", reader.tail(4)
+ print "The first five records look like this\n", reader.head()
+ if numpyOK:
+ print "First column:\n", reader[..., 0]
+ print "Row 4 & 5, first three cols\n", reader[4:6, :3]
+
+ ## ... Do a binary search for records --> idVar
+ print reader.get(4, "not found") # gets 1st record where id==4
+ print reader.get(4, "not found", full=True) # gets all records where id==4
+
+ ## ... Use a named tuple to get specific values
+ record = collections.namedtuple("_", " ".join(varNames))(*reader[0])
+ print record.id, record.educ, record.salary, record.bdate
+
+ reader.close()
+
+ ## ----- Convert file into .csv
+ savFileName = "in_unicode.sav"
+ csvFileName = "test_out.csv"
+ sav = SavReader(savFileName, ioLocale="dutch") # locale spec depends on OS
+ f = open(csvFileName, "wb")
+ with contextlib.nested(sav, f) as (sav, f):
writer = csv.writer(f)
for line in sav:
writer.writerow(line)
print "Done! Csv file written: %s" % f.name
-
+
## ---------------------------------
## SAV WRITER
## ---------------------------------
-
+
## ----- Write many rows
- records = [['Test1', 1, 1], ['Test2', 2, 1]]
- varNames = ['var1', 'v2', 'v3']
- varTypes = {'var1': 5, 'v2': 0, 'v3': 0}
- printTypesFile = {'var1': 'A41', 'v2': 'F3.1', 'v3': 'F5.1'}
- varLabels = {'var1': 'This is variable 1', 'v2': 'This is v2!'}
+ savFileName = "another_test.zsav" # .zsav = zlib compressed (--> v21 libs)
+ records = [['Test1', 1, 1, '2010-08-11'], ['Test2', 2, 1, '1910-01-12']]
+ varNames = ['var1', 'var2', 'var3', 'bdate']
+ varTypes = {'var1': 50, 'var2': 0, 'var3': 0, 'bdate': 10}
+ with SavWriter(savFileName, varNames, varTypes) as sav:
+ sav.writerows(records)
+ print "Done! %s" % sav.savFileName
+
+ savFileName = "employee data_OUT.sav"
+ varNames = ['id', 'gender', 'bdate', 'educ', 'jobcat', 'salary',
+ 'salbegin', 'jobtime', 'prevexp', 'minority']
+ varLabels = {'gender': 'guys/gals'}
+ varTypes = {'salary': 0, 'jobcat': 0, 'bdate': 0, 'minority': 0,
+ 'prevexp': 0, 'gender': 1, 'salbegin': 0, 'jobtime': 0,
+ 'educ': 0, 'id': 0}
+ varSets = {'SALARY': ['salbegin', 'salary'],
+ 'DEMOGR': ['gender', 'minority', 'educ']}
+ varAttributes = {'salary': {'DemographicVars': '1'},
+ 'jobcat': {'DemographicVars': '1'},
+ 'gender': {'Binary': 'Yes'},
+ 'educ': {'DemographicVars': '1'}}
+ fileAttributes = {'TheDate[2]': '10/21/2005',
+ 'RevisionDate[2]': '10/21/2005',
+ 'RevisionDate[1]': '10/29/2004'}
+ missingValues = {'educ': {'values': [1, 2, 3]}, 'gender': {'values': 'x'}}
+ records = [[1.0, 'm ', 11654150400.0, 15.0, 3.0,
+ 57000.0, 27000.0, 98.0, 144.0, 0.0],
+ [2.0, 'm ', 11852956800.0, 16.0, 1.0,
+ 40200.0, 18750.0, 98.0, 36.0, 0.0]]
+ with SavWriter(savFileName, varNames, varTypes,
+ varLabels=varLabels, varSets=varSets,
+ missingValues=missingValues, varAttributes=varAttributes,
+ fileAttributes=fileAttributes) as sav:
+ sav.writerows(records)
+ print "Done! %s" % sav.savFileName
+
+ ## ----- Write one row
+ # ... var1 is a 5-char string var, the others are numerical:
+ # ... formats, varLabels, valueLabels, missingValues etc. may also be
+ # ... None (default).
+ records = [['Test1', 1, 1, '2010-08-11'], ['Test2', 2, 1, '1910-01-12']]
+ varNames = ['var1', 'v2', 'v3', 'bdate']
+ varTypes = {'var1': 41, 'v2': 0, 'v3': 0, 'bdate': 0}
+ formats = {'var1': 'A41', 'v2': 'F3.1', 'v3': 'F5.1', 'bdate': 'EDATE40'}
+ missingValues = {'var1': {'values': ['Test1', 'Test2']},
+ 'v2': {'values': 1}}
+ varLabels = {'var1': 'This is variable 1',
+ 'v2': 'This is v2!',
+ 'bdate': 'dob'}
valueLabels = {'var1': {'Test1': 'Test1 value label',
'Test2': 'Test2 value label'},
- 'v2': {1: 'yes',
- 2: 'no'}}
- # ... printTypesFile, varLabels, valueLabels may also be None (default)
- savFileName = os.path.join(tempdir, "test.sav")
+ 'v2': {1: 'yes', 2: 'no'}}
+ # This also shows how date fields can be converted into spss dates.
+ # Spss dates are *stored* as the number of seconds since Oct 14, 1582, but
+ # *displayed* as <format>. In this case they are displayed as EDATE
+ # (European date, cf. ADATE = American date), ie. as dd.mm.yyyy
+ savFileName = "test1.sav"
with SavWriter(savFileName, varNames, varTypes,
- printTypesFile, varLabels, valueLabels) as sav:
- sav.writerows(records)
+ valueLabels, varLabels, formats) as sav:
+ pos = varNames.index("bdate")
+ for record in records:
+ record[pos] = sav.spssDateTime(record[pos], "%Y-%m-%d")
+ sav.writerow(record)
print "Done! %s" % sav.savFileName
- ## ----- Write one row
- savFileName = os.path.join(tempdir, "test2.sav")
- csvFileName = os.path.join(tempdir, "test.csv")
- varNames = ['var1', 'var2', 'var3']
- # ... var1 is a 50-char string var, the others are numerical:
- varTypes = {'var1': 50, 'var2': 0, 'var3': 0}
- if os.path.exists(csvFileName):
- with contextlib.nested(SavWriter(savFileName, varNames, varTypes),
- open(csvFileName, "rb")) as (sav, f):
- reader = csv.reader(f)
- header = reader.next()
- for row in reader:
- sav.writerow(row)
- print "Done! %s" % sav.savFileName
-
- ## ---------------------------------
- ## SAV DATA DICTIONARY READER
- ## ---------------------------------
-
- #savFileName = r"C:\Program Files\IBM\SPSS\Statistics\19\Samples\English\anorectic.sav"
- savFileName = "c:/program files/spss/employee data.sav"
- with SavDataDictionaryReader(savFileName) as spssDict:
- for varName in spssDict.varNames:
- print spssDict.getValueLabels(varName)[1]
- print spssDict.getVarLabel(varName)[1]
+ ## a test to check if non-ascii encodings work well
+ ## source: http://www.omniglot.com/language/phrases/hello.htm
+ greetings = [
+ [0, 'Arabic', u'\u0627\u0644\u0633\u0644\u0627\u0645\u0020\u0639\u0644\u064a\u0643\u0645'],
+ [1, 'Assamese', u'\u09a8\u09ae\u09b8\u09cd\u0995\u09be\u09f0'],
+ [2, 'Bengali', u'\u0986\u09b8\u09b8\u09be\u09b2\u09be\u09ae\u09c1\u0986\u09b2\u09be\u0987\u0995\u09c1\u09ae'],
+ [3, 'English', u'Greetings and salutations'],
+ [4, 'Georgian', u'\u10d2\u10d0\u10db\u10d0\u10e0\u10ef\u10dd\u10d1\u10d0'],
+ [5, 'Kazakh', u'\u0421\u04d9\u043b\u0435\u043c\u0435\u0442\u0441\u0456\u0437 \u0431\u0435'],
+ [6, 'Russian', u'\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435'],
+ [7, 'Spanish', u'\xa1Hola!'],
+ [8, 'Swiss German', u'Gr\xfcezi'],
+ [9, 'Thai', u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'],
+ [10, 'Walloon', u'Bondjo\xfb'],
+ [11, 'Telugu', u'\u0c0f\u0c2e\u0c02\u0c21\u0c40'],
+ ]
+ savFileName = "greetings_" + greetings[0][-1] + ".sav"
+ varNames = ['line', u'Bondjo\xfb', 'greeting']
+ varTypes = {'line': 0, u'Bondjo\xfb': 20, 'greeting': 50}
+ valueLabels = {u'Bondjo\xfb': {'Thai': u'\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35'}}
+ missingValues = {'line': {'lower': 0, 'upper': 9}, u'Bondjo\xfb': {'values': u'\xa1Hola!'}}
+ varLabels = {'greeting': u'\u0627\u0644\u0633\u0644\u0627\u0645\u0020\u0639\u0644\u064a\u0643\u0645'}
+ with SavWriter(savFileName, varNames, varTypes, valueLabels, varLabels,
+ missingValues=missingValues, ioUtf8=True) as sav:
+ sav.writerows(greetings)
+
+ with SavReader(savFileName, ioUtf8=True)as sav:
+ for lino, line in enumerate(sav):
+ print lino, "%s -- %s" % (line[1].strip(), line[2]), line
+
+ with SavHeaderReader(savFileName, ioUtf8=True)as spssDict:
wholeDict = spssDict.dataDictionary()
- print wholeDict["valueLabels"]
- print wholeDict["variableLabels"]
- spssDict.reportSpssDataDictionary(wholeDict)
+ #print wholeDict["varLabels"]["greeting"]
+ print unicode(spssDict)
+ pprint.pprint(wholeDict)
