Compare commits

...

18 Commits

Author SHA1 Message Date
Deon George
564b39b884 Fixes to compile on CentOS 8 - thanks to Torsten for providing this patch 2021-10-26 16:44:20 +11:00
Deon George
7685f64dc1 Update to TSM Client 8.1.7 2019-05-07 06:40:00 +10:00
Deon George
21e3ecd2cb Added docker image test for CI 2018-03-21 22:16:28 +11:00
Deon George
bfcea25d61 Revert "Changed to debian image - it's smaller"
SSL libraries missing and image not that much smaller.
2018-03-21 22:11:03 +11:00
Deon George
82841d8444 Changed to debian image - it's smaller 2018-03-21 19:58:48 +11:00
Deon George
2babbf6f77 Revert Dockerfile using built tsmpipe and example TSM library installs from IBM 2018-03-21 17:08:19 +11:00
Deon George
0f0d91147b Added back DOCKER_HOST to gitlab CI 2018-03-21 16:59:17 +11:00
Deon George
96ec7051aa Update SP password 2018-03-21 14:30:06 +11:00
Deon George
7bc743c0a3 Changed SP server address 2018-03-21 14:15:42 +11:00
Deon George
7723de32f5 Added application version reporting to SP 2018-03-21 00:13:36 +11:00
Deon George
4db350ebb0 Update to SP 8.1.4 2018-03-20 22:02:59 +11:00
Deon George
30fb54a485 Minor changes for automated build 2017-11-29 17:24:05 +11:00
Deon George
f404f9096e Updated Dockerfile to find compiled artifact 2017-11-29 15:41:09 +11:00
Deon George
c6694fe6de Changed source image for ci 2017-11-29 15:32:04 +11:00
Deon George
da6b7accbb Create docker image 2017-11-27 00:06:30 +11:00
Deon George
4152786d65 Update README.md 2017-03-01 21:15:17 +00:00
Deon George
5e3794054b Update README.md 2016-08-26 01:29:42 +00:00
Deon George
beef7a7196 Updated README 2016-08-26 11:26:00 +10:00
13 changed files with 160 additions and 47 deletions

4
.dockerignore Normal file
View File

@ -0,0 +1,4 @@
.git/
lib/
scripts/
test/

View File

@ -1,17 +1,19 @@
stages:
- build
- test
- build
- test
- docker
# Compile on CentOS 6.x, for some reason compiling on CentOS 7.x results in buffer corruption
# For some reason compiling on CentOS 6.x results in buffer corruption
build:6-7.1:
stage: build
image: leenooks/ci-tsm:6-7.1
image: ${CI_REGISTRY}/leenooks/ci-tsm:6-7.1
script:
- make -f Makefile.linux64
- make -f Makefile.linux64
only:
- master
- c6
tags:
- C
- C
- x86_64
artifacts:
name: "$CI_BUILD_NAME"
paths:
@ -19,39 +21,75 @@ build:6-7.1:
test:6-7.1:
stage: test
image: leenooks/ci-tsm:6-7.1
image: ${CI_REGISTRY}/leenooks/ci-tsm:6-7.1
dependencies:
- build:6-7.1
- build:6-7.1
script:
- cd test && ./test.sh
- cd test && ./test.sh
only:
- master
- c6
tags:
- C
- C
- x86_64
# Compile on CentOS 6.x, for some reason compiling on CentOS 7.x results in buffer corruption
build:7-7.1:
build:7-8.1:
stage: build
image: leenooks/ci-tsm:7-7.1
image: ${CI_REGISTRY}/leenooks/ci-tsm:7-8.1
script:
- make -f Makefile.linux64
- make -f Makefile.linux64
only:
- master
- master
tags:
- C
- C
- x86_64
artifacts:
name: "$CI_BUILD_NAME"
paths:
- tsmpipe
test:7-7.1:
test:7-8.1:
stage: test
image: leenooks/ci-tsm:7-7.1
image: ${CI_REGISTRY}/leenooks/ci-tsm:7-8.1
dependencies:
- build:7-7.1
- build:7-8.1
script:
- cd test && ./test.sh
- cd test && ./test.sh
only:
- master
- master
tags:
- C
- C
- x86_64
artifacts:
name: "$CI_BUILD_NAME"
paths:
- tsmpipe
docker:7-8.1:
stage: docker
image: docker:latest
services:
- docker:dind
before_script:
- docker info
- docker version
- echo "$CI_JOB_TOKEN" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
variables:
CACHETAG: build
VERSION: 8.1.7
DOCKER_HOST: tcp://docker:2375
dependencies:
- test:7-8.1
script:
- if [ -f init ]; then chmod 500 init; fi
- docker pull ${CI_REGISTRY_IMAGE}:${CACHETAG} || true
- docker build --cache-from ${CI_REGISTRY_IMAGE}:${CACHETAG} -t ${CI_REGISTRY_IMAGE}:${VERSION} -t ${CI_REGISTRY_IMAGE}:${CACHETAG} .
- docker run ${CI_REGISTRY_IMAGE}:${VERSION} -h
- docker push ${CI_REGISTRY_IMAGE}:${VERSION}
- docker push ${CI_REGISTRY_IMAGE}:${CACHETAG}
tags:
- docker
- x86_64
only:
- master

26
Dockerfile Normal file
View File

@ -0,0 +1,26 @@
# NAME deon/tsmpipe
# VERSION 8.1.7
FROM centos:7
# If using FTP source (change tar to not use z)
#RUN SOURCE_URL=ftp://public.dhe.ibm.com/storage/tivoli-storage-management/maintenance/client/v8r1/Linux/LinuxX86/BA/v812/8.1.2.0-TIV-TSMBAC-LinuxX86.tar \
# If using PPA source
RUN SOURCE_URL=http://yum.leenooks.vpn/docker/tsm/8.x/SP_CLIENT_8.1.7_LIN86_ML.tar.gz \
&& PREFIX=TSMCLI_LNX/tsmcli/linux86 \
&& CHARS=$(echo ${PREFIX} | awk -F"/" '{print NF}') \
&& curl -SL ${SOURCE_URL} | tar -zC /tmp/ --strip-components ${CHARS} -xf - ${PREFIX}/gskcrypt64-*.linux.x86_64.rpm ${PREFIX}/gskssl64-*.linux.x86_64.rpm ${PREFIX}/TIVsm-API64.x86_64.rpm \
&& yum install -y /tmp/*rpm && rm -rf /tmp/*rpm /var/cache/yum
RUN ln -s /tsm/dsm.sys /opt/tivoli/tsm/client/api/bin64/ \
&& ln -s /tsm/dsm.opt /opt/tivoli/tsm/client/api/bin64/ \
&& ln -s /tsm/dsmcert.kdb /opt/tivoli/tsm/client/api/bin64/ \
&& ln -s /tsm/dsmcert.sth /opt/tivoli/tsm/client/api/bin64/
COPY tsmpipe /usr/bin
VOLUME [ "/tsm" ]
# Starting
ENTRYPOINT [ "/usr/bin/tsmpipe" ]

View File

@ -1,6 +1,10 @@
# About
TSMPIPE is a Tivoli Storage Manager client, that enables you to take data from STDIN and store that in TSM as a BACKUP or ARCHIVE object.
TSMPIPE is a Tivoli Storage Manager (TSM) or now known as Spectrum Protect client, that enables you to take data from STDIN and store that in TSM as a BACKUP or ARCHIVE object.
By using this client, you dont need to do a dump to disk process, followed by using the Spectrum Protect Backup Archive Client. (So no more consuming disk space with old dumps!)
Restoring data, is the reverse step, restoring directly back to the application from Spectrum Protect.
# Using
In its simplest form, you can
```cat myfile.ext | tsmpipe -c [MORE OPTIONS]```
@ -13,5 +17,24 @@ Getting data back is the reverse
Or, back to your application
```tsmpipe -x [MORE OPTIONS] | myapp```
This client can also calculate a digest while backuping up data (like MD5, SHA1, etc) - so that you can verify that the dump out of the application matches what goes into
Spectrum Protect. For example:
```
myappdump > file
md5sum file
cat file | tsmpipe -cBs /example/ -f file -m md5
tsmpipe -xBs /example/ -f file > file2
md5sum file2
```
The MD5 calculation should match in all cases - including the MD5 printed by tsmpipe. See the build test script for more details.
# Building
Building TSMPIPE is hopefully relatively easy - it's currently only tested on Linux (specifically CentOS), but it should be buildable on other Spectrum Protect
supported platforms, however you may need to create an appropriate Makefile.
Here is a blog on building on Linux: https://thefrinkiac7.wordpress.com/data-protection/building-tsmpipe
# Help
For help, use tsmpipe -h
To test that you can connect to TSM, use tsmpipe -i
To test that you can connect to TSM, use tsmpipe -i

View File

@ -18,16 +18,17 @@ char *units(double size) {
int div = 0;
double rem = 0;
int base = 1024;
int s_size = 128;
char *s;
s = "\0";
s = malloc(128);
s = malloc(s_size);
if (s==NULL) {
perror("Arg, out of memory?");
exit(255);
}
memset(s,0x00,(sizeof s));
memset(s,0x00,s_size);
while (size >= base && div < (int)(sizeof SIZES / sizeof *SIZES)-1) {
if (verbose > 2)

View File

@ -72,14 +72,16 @@ dsmDate dsmStrToDate(char *s) {
char *dsmDateToStr(dsmDate date) {
char *s;
s = malloc(19);
int s_size = 32;
s = malloc(s_size);
if (s==NULL) {
perror("Arg, out of memory?");
exit(255);
}
memset(s,0x00,(sizeof s));
memset(s,0x00,s_size);
sprintf(s,"%04i-%02i-%02i %02i:%02i:%02i",
date.year,
@ -145,7 +147,7 @@ char *dsmObjnameToStr(dsmObjName objName) {
exit(255);
}
memset(s,0x00,(sizeof s));
memset(s,0x00,sizeof(dsmObjName));
sprintf(s,"%s%s%s",objName.fs,objName.hl,objName.ll);

View File

@ -133,10 +133,11 @@ double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long lon
#ifdef USE_DIGEST
char digest_str[EVP_MAX_MD_SIZE*2];
EVP_MD_CTX mdctx;
EVP_MD_CTX *mdctx;
const EVP_MD *md=NULL;
unsigned char md_value[EVP_MAX_MD_SIZE];
unsigned int md_len, i;
mdctx = NULL;
if (digest) {
OpenSSL_add_all_digests();
@ -229,8 +230,8 @@ double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long lon
#ifdef USE_DIGEST
if (digest) {
EVP_MD_CTX_init(&mdctx);
EVP_DigestInit_ex(&mdctx, md, NULL);
mdctx = EVP_MD_CTX_new();
EVP_DigestInit_ex(mdctx, md, NULL);
}
#endif
@ -250,7 +251,7 @@ double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long lon
#ifdef USE_DIGEST
if (digest)
EVP_DigestUpdate(&mdctx, buffer, nbytes);
EVP_DigestUpdate(mdctx, buffer, nbytes);
#endif
dataBlk.bufferLen = nbytes;
@ -289,8 +290,8 @@ double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long lon
#ifdef USE_DIGEST
if (digest) {
EVP_DigestFinal_ex(&mdctx, md_value, &md_len);
EVP_MD_CTX_cleanup(&mdctx);
EVP_DigestFinal_ex(mdctx, md_value, &md_len);
EVP_MD_CTX_free(mdctx);
for(i=0; i<md_len; i++)
sprintf(digest_str+(i*2), "%02x", md_value[i]);

View File

@ -23,7 +23,7 @@ char *tsm_printerr(dsUint32_t dsmHandle, dsInt16_t rc) {
char rcStr[DSM_MAX_RC_MSG_LENGTH];
char *s;
s = malloc(sizeof(rcStr));
s = malloc(DSM_MAX_RC_MSG_LENGTH+5);
if (rc == DSM_RC_WILL_ABORT) {
dsUint16_t reason;
@ -79,6 +79,7 @@ dsUint32_t tsm_initsess(char *options, char *password) {
dsInt16_t rc=0;
dsUint32_t dsmHandle=0;
dsmApiVersionEx applApi;
dsmAppVersion pipeApi;
dsmInitExIn_t initIn;
dsmInitExOut_t initOut;
optStruct dsmOpt;
@ -95,6 +96,13 @@ dsUint32_t tsm_initsess(char *options, char *password) {
applApi.level = DSM_API_LEVEL;
applApi.subLevel = DSM_API_SUBLEVEL;
memset(&pipeApi,0x00,sizeof(dsmAppVersion));
pipeApi.stVersion = appVersionVer;
pipeApi.applicationVersion = TSMPIPE_VERSION;
pipeApi.applicationRelease = TSMPIPE_RELEASE;
pipeApi.applicationLevel = TSMPIPE_LEVEL;
pipeApi.applicationSubLevel = TSMPIPE_SUBLEVEL;
memset(&initIn,0x00,sizeof(dsmInitExIn_t));
initIn.stVersion = dsmInitExInVersion;
initIn.apiVersionExP = &applApi;
@ -106,6 +114,7 @@ dsUint32_t tsm_initsess(char *options, char *password) {
initIn.options = options;
initIn.userNameP = NULL;
initIn.userPasswordP = NULL;
initIn.appVersionP = &pipeApi;
//initIn.dirDelimiter = dirDelimiter;
//initIn.useUnicode = useUnicode;
//initIn.bEncryptKeyEnabled = encrypt;
@ -171,8 +180,11 @@ int tsm_sessioninfo(dsUint32_t dsmHandle) {
dsmQueryApiVersionEx(&apiLibVer);
printf("Application Version:\n");
printf(" TSMPIPE Version: %s\n",
_TSMPIPE_VERSION);
printf(" TSMPIPE Version: %d.%d.%d.%d\n",
TSMPIPE_VERSION,
TSMPIPE_RELEASE,
TSMPIPE_LEVEL,
TSMPIPE_SUBLEVEL);
printf(" TSMPIPE API Version: %d.%d.%d.%d\n",
DSM_API_VERSION,
DSM_API_RELEASE,

View File

@ -0,0 +1 @@
COMPRESSAlways Y

View File

@ -3,7 +3,7 @@ SErvername test
COMMmethod TCPip
MANAGEDServices schedule
TCPPort 1500
TCPServeraddress tsm.leenooks.vpn
TCPServeraddress tsm.leenooks.net
PASSWORDAccess generate
SCHEDMode polling
@ -16,3 +16,5 @@ SErvername test
TCPBuffsize 32
TCPWindowsize 64
SSL No
PASSWORDDIR /tsm

View File

@ -8,7 +8,7 @@ VERBOSE=${VERBOSE:-""}
# Create TEST File
MD5=$(dd if=/dev/urandom bs=10240 count=1024 2>/dev/null |tee ${FILE} |md5sum |awk '{print $1}')
SIZE=$(ls -l ${FILE}|awk '{print $5}')
TSMOPT="-VIRTUALNODENAME=TEST -PASSWORD=TEST -COMPRESSALWAYS=YES"
TSMOPT="-VIRTUALNODENAME=TEST -PASSWORD=password -COMPRESSALWAYS=YES"
TSMPIPE=../tsmpipe
@ -69,7 +69,7 @@ cat ${FILE} | ${TSMPIPE} -Acm MD5 -s /test -f ${FILE} ${VERBOSE} -l ${SIZE} -D"T
DATE=$(date +%m%d%Y:%H%M%S)
# List it
echo "+ ARCIHVE: LIST FILE!"
echo "+ ARCHIVE: LIST FILE!"
${TSMPIPE} -Ats /test -f ${FILE} -O"${TSMOPT}"
# Get it back
@ -83,11 +83,11 @@ else
fi
# Purge the last one
echo "+ ARCHIVE: DELETE!"
echo "+ ARCHIVE: DELETE!"
${TSMPIPE} -Ads /test -f ${FILE} -n ${DATE} ${VERBOSE} -O"${TSMOPT}"
# List it
echo "+ ARCHIVE: LIST FILE AFTER DELETE!"
echo "+ ARCHIVE: LIST FILE AFTER DELETE!"
${TSMPIPE} -Ats /test -f ${FILE} -O"${TSMOPT}"
echo "+++ END ARCHIVE ++++!"

View File

@ -67,7 +67,7 @@ int copy_env(const char *from, const char *to) {
void usage(void) {
fprintf(stderr,
"tsmpipe %s, usage:\n"
"tsmpipe %d.%d.%d.%d, usage:\n"
"\n"
"tsmpipe [-i|-p]|[[-A|-B|-U] [-c|-x|-d|-g|-t] -s fsname -f filepath [-l len] ...]\n"
" -i Show session information\n"
@ -102,7 +102,7 @@ void usage(void) {
#endif
" -v Verbose. More -v's gives more verbosity\n"
" -V Verbose information on TSM transfer\n"
,_TSMPIPE_VERSION);
,TSMPIPE_VERSION,TSMPIPE_RELEASE,TSMPIPE_LEVEL,TSMPIPE_SUBLEVEL);
exit(0);
}

View File

@ -1,4 +1,7 @@
#define _TSMPIPE_VERSION "1.6.6"
#define TSMPIPE_VERSION 1
#define TSMPIPE_RELEASE 6
#define TSMPIPE_LEVEL 6
#define TSMPIPE_SUBLEVEL 0
#define INPUTLEN 1025
#define ACTION_INFO 1