From 2a9f0c0aacc08b2cdfe79d72004134202b9d1f8b Mon Sep 17 00:00:00 2001 From: Deon George Date: Fri, 20 Jun 2014 22:51:18 +1000 Subject: [PATCH] Update FS with object sizes and print hashes --- lib/dsmlist.c | 44 +++++++++++++++++----- lib/dsmobjects.c | 4 +- lib/dsmsendrecv.c | 94 ++++++++++++++++++++++++++++------------------- tsmpipe.c | 55 ++++++++++++++++++++------- tsmpipe.h | 15 ++++++-- 5 files changed, 146 insertions(+), 66 deletions(-) diff --git a/lib/dsmlist.c b/lib/dsmlist.c index 6f3d6e5..c8d3fcf 100644 --- a/lib/dsmlist.c +++ b/lib/dsmlist.c @@ -19,26 +19,33 @@ int tsm_listfile_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) { char stor[2]; char state[3]; char ced[4]; + char objInfo[DSM_MAX_OBJINFO_LENGTH]; if (userdata != NULL ) { - fprintf(stderr, "tsm_listfile_cb: Internal error: userdate != NULL"); + fprintf(stderr, "tsm_listfile_cb: Internal error: userdata != NULL"); return -1; } if (qType == qtArchive) { - if (! headerPrinted++) - printf("%40s %2s %s %3s %19s %19s %10s %10s %s\n", + if (! headerPrinted++) { + printf("%40s %2s %s %3s %19s %19s %10s %10s %10s", "NAME", "ST", "L", "CED", "BACKUP", "EXPIRE", - "EST SIZE", + "SIZE", "ID", "DESC" ); +#ifdef USE_DIGEST + printf(" %s","OBJINFO"); +#endif + printf("\n"); + } + qryRespArchiveData respArchive; memset(&respArchive,0x00,sizeof(qryRespArchiveData)); qryRespArchiveData *qr = (void *) qResp->bufferPtr; @@ -68,22 +75,29 @@ int tsm_listfile_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) { strcat(ced,respArchive.clientDeduplicated ? "D" : "-"); // The Object Status - printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%u|%s\n",dsmObjnameToStr(respArchive.objName),state,stor,ced,dsmDateToStr(respArchive.insDate),respArchive.expDate.year ? dsmDateToStr(respArchive.expDate) : "",dsmSizeToNum(respArchive.sizeEstimate),respArchive.objId.hi,respArchive.objId.lo,respArchive.descr); + printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%u|%s",dsmObjnameToStr(respArchive.objName),state,stor,ced,dsmDateToStr(respArchive.insDate),respArchive.expDate.year ? dsmDateToStr(respArchive.expDate) : "",dsmSizeToNum(respArchive.sizeEstimate,bTrue),respArchive.objId.hi,respArchive.objId.lo,respArchive.descr); } else if (qType == qtBackup) { - if (! headerPrinted++) - printf("%40s %2s %s %3s %19s %19s %10s %s\n", + if (! headerPrinted++) { + printf("%40s %2s %s %3s %19s %19s %10s %10s", "NAME", "ST", "L", "CED", "BACKUP", "EXPIRE", - "EST SIZE", + "SIZE", "ID" ); +#ifdef USE_DIGEST + printf(" %s","OBJINFO"); +#endif + printf("\n"); + } + qryRespBackupData respBackup; + memset(&respBackup,0x00,sizeof(qryRespBackupData)); qryRespBackupData *qr = (void *) qResp->bufferPtr; respBackup = *qr; @@ -117,13 +131,25 @@ int tsm_listfile_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) { strcat(ced,respBackup.clientDeduplicated ? "D" : "-"); - printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%u\n",dsmObjnameToStr(respBackup.objName),state,stor,ced,dsmDateToStr(respBackup.insDate),respBackup.expDate.year ? dsmDateToStr(respBackup.expDate) : "",dsmSizeToNum(respBackup.sizeEstimate),respBackup.objId.hi,respBackup.objId.lo); +#ifdef USE_DIGEST + // Object Info + memset(&objInfo,0x00,sizeof(objInfo)); + if (respBackup.objInfolen) + strncat(objInfo,respBackup.objInfo,respBackup.objInfolen); +#endif + + printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%8u",dsmObjnameToStr(respBackup.objName),state,stor,ced,dsmDateToStr(respBackup.insDate),respBackup.expDate.year ? dsmDateToStr(respBackup.expDate) : "",dsmSizeToNum(respBackup.sizeEstimate,bTrue),respBackup.objId.hi,respBackup.objId.lo); } else { fprintf(stderr,"tsm_listfile_cb: Internal error: Unknown qType %d\n",qType); return -1; } +#ifdef USE_DIGEST + printf("|%s",objInfo); +#endif + printf("\n"); + return 1; } diff --git a/lib/dsmobjects.c b/lib/dsmobjects.c index 08c568a..a8688e6 100644 --- a/lib/dsmobjects.c +++ b/lib/dsmobjects.c @@ -152,7 +152,7 @@ char *dsmObjnameToStr(dsmObjName objName) { return s; } -double dsmSizeToNum(dsStruct64_t dsStruct64) { +double dsmSizeToNum(dsStruct64_t dsStruct64,dsBool_t inMB) { unsigned long long filesize=0; filesize = 0x00; @@ -162,7 +162,7 @@ double dsmSizeToNum(dsStruct64_t dsStruct64) { filesize |= dsStruct64.lo; // Return number in MB - return (float)filesize/1024/1024; + return (float) inMB ? filesize/1024/1024 : filesize; } void debugLog(int level, _IO_FILE *output, char *message, int die) { diff --git a/lib/dsmsendrecv.c b/lib/dsmsendrecv.c index 6a65c0a..9094cd9 100644 --- a/lib/dsmsendrecv.c +++ b/lib/dsmsendrecv.c @@ -1,17 +1,5 @@ /* - * The recommended buffer size is n*TCPBUFFLEN - 4 bytes. - * To get your buffer size, do: dsmc query options|grep TCPBUF - * 32kB seems to be the new default, 31kB was the old. - * - * An additional factor is the pipe buffer size. Since we don't do threading - * (yet), we hide a little bit of latency if we don't read more than the pipe - * buffer can hold at a time. On Linux 2.6 this is 64kB. - * - * So, I would recommend a BUFLEN of 64kB if your TCPBUFLEN is above the - * 64kB + 4 bytes limit or if your TCPBUFLEN is lower, the - * n*TCPBUFFLEN - 4 bytes that gets you closest to 64kB. - * - * For a default tuned TSM client on Linux, BUFLEN should thus be 64*1024-4. + * This function controls our communication with TSM */ #define BUFLEN (64*1024-4) @@ -28,10 +16,6 @@ #include "../tsmpipe.h" -#ifdef USE_DIGEST -#include -#endif - int tsm_matchone_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) { struct matchone_cb_data *cbdata = userdata; @@ -123,9 +107,9 @@ int tsm_regfs(dsUint32_t dsmHandle, char *fsname) { /* Send data to TSM for storage */ #ifdef USE_DIGEST -int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, char *digest) { +double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBool_t summary, char *digest) { #else -int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype) { +double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBool_t summary) { #endif extern int verbose; char *buffer; @@ -147,6 +131,7 @@ int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long l } #ifdef USE_DIGEST + char digest_str[EVP_MAX_MD_SIZE*2]; EVP_MD_CTX mdctx; const EVP_MD *md=NULL; unsigned char md_value[EVP_MAX_MD_SIZE]; @@ -162,6 +147,7 @@ int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long l return 0; } } + #endif // Register our filespace @@ -296,33 +282,45 @@ int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long l return 0; } - if (verbose) { - printf("Total bytes sent: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalBytesSent)); - - if (dsmEndSendObjExOut.objDeduplicated) - printf("Deduplicated size: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalDedupSize)); - if (dsmEndSendObjExOut.objCompressed) - printf("Compressed size: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalCompressSize)); - if (dsmSizeToNum(dsmEndSendObjExOut.totalLFBytesSent)) - printf("LAN-free bytes sentsize: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalLFBytesSent)); - - if (dsmEndSendObjExOut.encryptionType) { - printf("Encryption: %s\n",dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_CLIENTENCRKEY ? "CLIENTENCRKEY" : dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_USER ? "USER" : "NO"); - printf("Encryption Strength: %s\n",dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_AES_128BIT ? "AES_128BIT" : dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_DES_56BIT ? "DES_56BIT" : "NONE"); - } - } - #ifdef USE_DIGEST if (digest) { EVP_DigestFinal_ex(&mdctx, md_value, &md_len); EVP_MD_CTX_cleanup(&mdctx); - for(i=0; i> 32; + fsUpdateP.occupancy.lo = size & ~0U; + fsUpdateP.capacity = fsUpdateP.occupancy; + + rc = dsmUpdateFS(dsmHandle,fsname,&fsUpdateP,DSM_FSUPD_OCCUPANCY | DSM_FSUPD_CAPACITY); + + if (rc != DSM_RC_OK) { + tsm_printerr(dsmHandle,rc,"tsm_updatefs: dsmUpdateFS failed"); + return 0; + } + + return 1; +} diff --git a/tsmpipe.c b/tsmpipe.c index faff581..740b924 100644 --- a/tsmpipe.c +++ b/tsmpipe.c @@ -17,8 +17,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - Original Idea Copyright (c) 2006,2007 HPC2N, Umeå University, Sweden - Modifications Copyright (c) 2012-2013 by Deon George + Original idea International Business Machines redook "Using ADSM to Back Up +Databases", published 1998. + + Some adaption from HPC2N, UmeÃ¥ University, Sweden, 2006/7 + + This version Copyright (c) 2012-2014 by Deon George */ /* Enable Large File Support stuff */ @@ -65,21 +69,24 @@ void usage(void) { fprintf(stderr, "tsmpipe %s, usage:\n" "\n" - "tsmpipe [-i]|[[-A|-B] [-c|-x|-d|-t] -s fsname -f filepath [-l len] ...]\n" + "tsmpipe [-i]|[[-A|-B|-U] [-c|-x|-d|-t] -s fsname -f filepath [-l len] ...]\n" " -i Show session information:\n" - " -A and -B are mutually exclusive:\n" + " -A, -B and -U are mutually exclusive:\n" " -A Use Archive objects\n" " -B Use Backup objects\n" + " -U Update FS with utilisation info\n" " -c, -x, -d and -t are mutually exclusive:\n" " -c Create: Read from stdin and store in TSM\n" " -x eXtract: Recall from TSM and write to stdout\n" " -d Delete: Delete object from TSM\n" " -t lisT: Print filelist to stdout\n" " -s and -f are required arguments for (-A/ -B operations):\n" - " -s fsname Name of filesystem in TSM\n" + " -s fsname Name of filespace in TSM\n" " -f filepath Path to file within filesystem in TSM\n" " -l length Length of object to store. If guesstimating, too large\n" " is better than too small\n" + " -L length Length of object to store. But also update the FS with\n" + " the size of the object sent\n" " -D desc Description of archive object\n" " -P pitdate PITDate (mmddYYYY[:HHMMSS]) (BACKUP Objects)\n" " -n insdate Insert Date lower bound (mmddYYYY[:HHMMSS]) (ARCHIVE Objects)\n" @@ -91,6 +98,7 @@ void usage(void) { " -m digest Calculate digest for data being stored in TSM, eg: md5, sha1...\n" #endif " -v Verbose. More -v's gives more verbosity\n" + " -V Verbose information on TSM transfer\n" ,_TSMPIPE_VERSION); exit(1); @@ -108,6 +116,7 @@ int main(int argc, char *argv[]) { extern char *optarg; int action=0; char *space=NULL, *filename=NULL, *lenstr=NULL, *desc=NULL, *pitdate=NULL, *options=NULL, *expdate_low=NULL, *expdate_high=NULL, *insdate_low=NULL, *insdate_high=NULL; + dsBool_t summary=bFalse; off_t length; dsUint32_t dsmHandle; dsmQueryType qType=0xff; // We set this to an unlikely value, it should be set correctly via our getopt @@ -115,11 +124,12 @@ int main(int argc, char *argv[]) { qryBackupData qbData; dsmObjName objName; int rc=0; + double stored=0; #ifdef USE_DIGEST char *digest=NULL; #endif - while ((c = getopt(argc, argv, "hiABcxdtve:E:f:l:m:n:N:s:D:O:P:")) != -1) { + while ((c = getopt(argc, argv, "hiABcxdtUvVe:E:f:l:L:m:n:N:s:D:O:P:")) != -1) { switch(c) { case 'h': usage(); break; case 'A': qType = qtArchive; break; @@ -130,11 +140,14 @@ int main(int argc, char *argv[]) { case 'x': if (action != 0) usage_action(); action = ACTION_EXTRACT; break; case 'd': if (action != 0) usage_action(); action = ACTION_DELETE; break; case 't': if (action != 0) usage_action(); action = ACTION_LIST; break; + case 'U': if (action != 0) usage_action(); action = ACTION_UPDATE; break; case 'v': verbose++; break; + case 'V': summary = bTrue; break; case 's': space = optarg; break; case 'f': filename = optarg; break; case 'l': lenstr = optarg; break; + case 'L': lenstr = optarg; if (action != ACTION_CREATE) usage_action(); action = ACTION_CREATE_UPDATE; break; case 'D': desc = optarg; break; #ifdef USE_DIGEST @@ -179,18 +192,18 @@ int main(int argc, char *argv[]) { exit(1); } - if (! filename) { + if (! filename && action != ACTION_UPDATE) { fprintf(stderr, "tsmpipe: ERROR: Must give -f filename\n"); exit(1); } - if (action == ACTION_CREATE && ! lenstr) { - fprintf(stderr, "tsmpipe: ERROR: Must give -l length with -c\n"); + if ((action == ACTION_CREATE || action == ACTION_CREATE_UPDATE || action == ACTION_UPDATE) && ! lenstr) { + fprintf(stderr, "tsmpipe: ERROR: Must give -l|-L length with -c | -u\n"); exit(1); } - if (action != ACTION_CREATE && lenstr) { - fprintf(stderr, "tsmpipe: ERROR: -l length useless without -c\n"); + if ((action != ACTION_CREATE && action != ACTION_CREATE_UPDATE && action != ACTION_UPDATE) && lenstr) { + fprintf(stderr, "tsmpipe: ERROR: -l|-L length useless without -c | -u\n"); exit(1); } @@ -227,6 +240,7 @@ int main(int argc, char *argv[]) { // If we are backing up or archiving case ACTION_CREATE: + case ACTION_CREATE_UPDATE: debugLog(2,stderr,"tsmpipe: CREATE Operation",0); length = atof(lenstr); @@ -234,10 +248,17 @@ int main(int argc, char *argv[]) { debugLog(0,stderr,"tsmpipe: ERROR: Provide positive length, overestimate if guessing",1); #ifdef USE_DIGEST - rc = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait),digest); + stored = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait),summary,digest); #else - rc = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait)); + stored = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait),summary); #endif + rc = stored ? 1 : 0; + + if (rc && action == ACTION_CREATE_UPDATE && qType==qtBackup) { + debugLog(2,stderr,"tsmpipe: UPDATE FS Operation",0); + rc = tsm_updatefs(dsmHandle,space,stored); + } + break; case ACTION_DELETE: @@ -327,6 +348,14 @@ int main(int argc, char *argv[]) { } break; + case ACTION_UPDATE: + length = atof(lenstr); + if (length <= 0) + debugLog(0,stderr,"tsmpipe: ERROR: Provide positive length",1); + + rc = tsm_updatefs(dsmHandle,space,length); + break; + default: fprintf(stderr,"tsmpipe: UNKNOWN Operation %d",action); exit(2); diff --git a/tsmpipe.h b/tsmpipe.h index 5ecd561..0c10262 100644 --- a/tsmpipe.h +++ b/tsmpipe.h @@ -1,14 +1,20 @@ -#define _TSMPIPE_VERSION "1.5.5" +#define _TSMPIPE_VERSION "1.6.0" #define ACTION_INFO 1 #define ACTION_CREATE 2 #define ACTION_EXTRACT 3 #define ACTION_DELETE 4 #define ACTION_LIST 5 +#define ACTION_UPDATE 6 +#define ACTION_CREATE_UPDATE 7 // If you want to use MD5/SHA1 calculations as the data goes in here, ensure this is defined. #define USE_DIGEST +#ifdef USE_DIGEST +#include +#endif + typedef int (*tsm_query_callback)(dsmQueryType, DataBlk *, void *); struct matchone_cb_data { int numfound; dsStruct64_t objId; dsUint32_t copyGroup; }; @@ -26,13 +32,14 @@ extern int tsm_deletefile (dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveD extern int tsm_listfile (dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveData qaData, qryBackupData qbData); extern int tsm_restorefile(dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveData qaData, qryBackupData qbData); #ifdef USE_DIGEST -extern int tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, char *digest); +extern double tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBool_t summary, char *digest); #else -extern int tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype); +extern double tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBoot_t summary); #endif extern int tsm_sessioninfo(dsUint32_t dsmHandle); +extern int tsm_updatefs(dsUint32_t dsmHandle, char *fsname, long long size); extern char *dsmDateToStr(dsmDate date); extern char *dsmObjnameToStr(dsmObjName objName); -extern double dsmSizeToNum(dsStruct64_t dsStruct64); +extern double dsmSizeToNum(dsStruct64_t dsStruct64,dsBool_t inMB);