تقارير عن حالة التخزين باستخدام R. Parallel computing ، الرسوم البيانية ، xlsx ، البريد الإلكتروني وكل هذا

cd 'd:\Navisphere CLI\' $csv = "D:\VNX-IP.csv" $Filter1 = "name1" $Filter2 = "name2" $Filter3 = "name3" $Data = import-csv $csv -Delimiter ';' | Where {$_.cl -EQ $Filter1 -Or $_.cl -EQ $Filter2 -Or $_.cl -EQ $Filter3} | Sort-Object -Property @{Expression={$_.cl}; Ascending=$true}, @{Expression={$_.Name} ;Ascending=$true} #$Filter1 = "nameOfcl" #$Data = import-csv $csv -Delimiter ';' | Where {$_.Name -EQ $Filter1} $Data | select Name,IP,cl $yStart = (Get-Date).AddDays(-30).ToString('yyyy') $yEnd = (Get-Date).ToString('yyyy') $mStart = (Get-Date).AddDays(-30).ToString('MM') $mEnd = (Get-Date).ToString('MM') $dStart = (Get-Date).AddDays(-30).ToString('dd') $dEnd = (Get-Date).ToString('dd') #$start = (Get-Date).AddDays(-3).ToString('MM\/dd\/yy') #$end = (Get-Date).ToString('MM\/dd\/yy') $i = 1 $table = ForEach ($row in $Data) { Write-Host $row.Name -ForegroundColor "Yellow" Write-Host "SP A" Write-Host (Get-Date).ToString('HH:mm:ss') $txt = .\NaviSECCli.exe -scope 0 -h $row.newA -user myusername -password mypassword getlog -date $mStart/$dStart/$yStart $mEnd/$dEnd/$yEnd | Select-String -Pattern "\(820\)","\(803\)","\(801\)","\(920\)","\(901\)" ForEach ($n in $txt) { $x = $n -Split(' ') $disk = $x[3] + "_" + $x[5] + "_" + $x[7].Split("(")[0] $sn = (.\NaviSECCli.exe -scope 0 -h $row.newA -user myusername -password mypassword getdisk $disk -serial)[1] | %{$_ -replace "Serial Number: ",""} | %{$_ -replace "State: ",""} | %{$_ -replace " ",""} New-Object PSObject -Property @{ i = $i cl = $row.cl Storage = $row.Name SP = "A" Date = $x[0] Time = $x[1] Disk = $disk Error = (($n -Split('\['))[0] -Split('\)'))[1].Trim() eCode = (($n -Split('\('))[1] -Split('\)'))[0] SN = $sn } $i = $i + 1 } Write-Host "SP B" Write-Host (Get-Date).ToString('HH:mm:ss') $txt = .\NaviSECCli.exe -scope 0 -h $row.newB -user myusername -password mypassword getlog -date $mStart/$dStart/$yStart $mEnd/$dEnd/$yEnd | Select-String -Pattern "\(820\)","\(803\)","\(801\)","\(920\)","\(901\)" ForEach ($n in $txt) { $x = $n -Split(' ') $disk = $x[3] + "_" + $x[5] + "_" + $x[7].Split("(")[0] $sn = (.\NaviSECCli.exe -scope 0 -h $row.newA -user myusername -password mypassword getdisk $disk -serial)[1] | %{$_ -replace "Serial Number: ",""} | %{$_ -replace "State: ",""} | %{$_ -replace " ",""} New-Object PSObject -Property @{ i = $i cl = $row.cl Storage = $row.Name SP = "B" Date = $x[0] Time = $x[1] Disk = $disk Error = (($n -Split('\['))[0] -Split('\)'))[1].Trim() eCode = (($n -Split('\('))[1] -Split('\)'))[0] SN = $sn } $i = $i + 1 } Write-Host " " } $table | select i,cl,Storage,SP,Date,Time,Disk,Error,eCode,SN | Export-Csv -Path 'd:\VNX-Errors.csv' -NoTypeInformation -UseCulture -Encoding UTF8 

 # A tibble: 83 x 9 Name IP cl type newA newB oldA oldB cntIP <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> 1 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 2 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 3 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 4 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 5 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 6 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 7 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 8 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 9 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ 10 XXX 10.***.**~ XclNam~ 5300-1 10.201.1~ 10.201.1~ 10.***.*~ 10.***.*~ 10.***.*~ # ... with 73 more rows 

 numCores <- detectCores() registerDoParallel(numCores) 

 #      VNX   ip   . #      ,      #   ,      dataframe #  %dopar%   . #    ,   system.time   , #    %dopar%  %do%.      4-5. # system.time({ errors_df <- foreach(i = 1:nrow(VNX_ip), .combine = rbind, .packages = "tidyverse") %dopar% { errors_raw <- system( paste( "NaviSECCli.exe -scope 0 -h", VNX_ip$cntIP[i], "-user myusername -password mypassword getlog -date", bigPeriodForm, currDateForm ), intern = TRUE ) %>% str_subset(pattern = regex(paste0(errorNumbers, collapse = "|"))) #      ,       if (length(errors_raw) > 0) { #     #       , #     , #       . errorsDescr <- errors_raw %>% gsub("(.*\\) )(.*)(\\s+\\[.*)", "\\2", x = .) %>% trimws() %>% gsub('([[:punct:]])|\\s+', '_', .) #           errors <- errors_raw %>% str_split(pattern = "\\s+", simplify = T) %>% as_tibble() %>% mutate(Disk = paste0(V4, "_", V6, "_", V8) %>% gsub( pattern = "\\([0-9]{3}\\)", replacement = "", x = .) ) #  dataframe    data_frame(cl = VNX_ip$cl[i], Storage = VNX_ip$Name[i], Date = errors$V1 %>% as.Date(format = "%m/%d/%Y"), Time = errors$V2, Disk = errors$Disk, Error = errorsDescr, eCode = errors$V8 %>% str_extract(paste0(errorNumbers, collapse = "|")) %>% str_extract("[0-9]+")) %>% mutate(DateTime = as.POSIXct(paste(Date, Time), format = "%Y-%m-%d %H:%M:%S")) } } # }) 

 > errors_df # A tibble: 2,705 x 8 cl Storage Date Time Disk Error eCode DateTime <chr> <chr> <date> <chr> <chr> <chr> <chr> <dttm> 1 XclNam~ XStorageN~ 2019-07-18 12:09:~ 0_1_3 Soft_SCSI_~ 801 2019-07-18 12:09:55 2 XclNam~ XStorageN~ 2019-07-18 15:09:~ 0_1_3 Soft_SCSI_~ 801 2019-07-18 15:09:56 3 XclNam~ XStorageN~ 2019-07-18 16:28:~ 0_1_3 Soft_SCSI_~ 801 2019-07-18 16:28:50 4 XclNam~ XStorageN~ 2019-07-19 06:36:~ 0_1_6 Soft_SCSI_~ 801 2019-07-19 06:36:39 5 XclNam~ XStorageN~ 2019-07-19 20:57:~ 0_1_6 Soft_Media~ 820 2019-07-19 20:57:35 6 XclNam~ XStorageN~ 2019-07-22 11:00:~ 0_2_~ Soft_SCSI_~ 801 2019-07-22 11:00:43 7 XclNam~ XStorageN~ 2019-07-22 11:00:~ 0_2_~ Soft_SCSI_~ 801 2019-07-22 11:00:44 8 XclNam~ XStorageN~ 2019-07-22 12:02:~ 0_2_~ Soft_SCSI_~ 801 2019-07-22 12:02:31 9 XclNam~ XStorageN~ 2019-07-23 23:29:~ 0_3_8 Soft_SCSI_~ 801 2019-07-23 23:29:49 10 XclNam~ XStorageN~ 2019-07-13 00:01:~ 0_3_9 Soft_SCSI_~ 801 2019-07-13 00:01:46 # ... with 2,695 more rows 

 myErrorStats <- function(data, period, orderColname = quo(Soft_Media_Error)) { data %>% filter(Date > period) %>% group_by(cl, Storage, Disk, Error) %>% summarise(count = n()) %>% spread(Error, count, fill = 0) %>% arrange(desc(!!orderColname)) } 

 > errorsBigPeriod # A tibble: 77 x 7 cl Storage Disk Hard_SCSI_Bus_E~ Recommend_Disk_~ Soft_Media_Error <chr> <chr> <chr> <dbl> <dbl> <dbl> 1 XclN~ XStora~ 1_1_~ 0 1 64 2 XclN~ XStora~ 0_2_5 0 0 29 3 XclN~ XStora~ 1_1_~ 0 1 29 4 XclN~ XStora~ 0_3_2 0 0 27 5 XclN~ XStora~ 0_3_~ 1 0 25 6 XclN~ XStora~ 1_3_5 0 1 23 7 XclN~ XStora~ 0_2_9 0 0 21 8 XclN~ XStora~ 0_3_4 0 0 14 9 XclN~ XStora~ 0_1_~ 0 0 14 10 XclN~ XStora~ 1_0_1 0 0 12 # ... with 67 more rows, and 1 more variable: Soft_SCSI_Bus_Error <dbl> 

 ####   #### #       errorsTotal <- errors_df %>% group_by(Date, Error) %>% summarise(count = n()) %>% # spread(Error, count, fill = 0) %>% arrange(desc(Date)) #        . #       ,   . plot <- errorsTotal %>% ggplot(aes(x = Date, y = count, fill = Error)) + geom_bar(stat = "identity", width = 0.5, color = "grey") + theme_minimal() + # theme(legend.position="top") + scale_color_grey() + labs(title = "  EMC VNX", subtitle = "        ", fill = " ") + xlab("") + ylab("   ") + scale_fill_brewer(palette = "Spectral") + facet_grid(rows = vars(factor( Error, levels = c( "Soft_SCSI_Bus_Error", "Soft_Media_Error", "Hard_SCSI_Bus_Error", "Recommend_Disk_Replacement" ) ))) # #   # plot #   png,     plot_filePath <- file.path("results", "plot.png") #    png     ggsave(filename = plot_filePath, plot = plot) 

 #### ENV #### #         (    system32) setwd("C:\\Scripts\\VNX_disks_check/") #   library(tidyverse) library(lubridate) library(zoo) library(stringi) library(xlsx) library(mailR) library(foreach) library(doParallel) #### CONST #### #            vnxIPfilePath <- file.path("data", "VNX-IP.csv") #     bigPeriod <- Sys.Date() - 30 #     smallPeriod <- Sys.Date() - 3 #    productCls <- c("name1", "name2", "name3") #   IP VNX. #         , #   ip        . VNX_ip <- vnxIPfilePath %>% read_csv2( col_types = cols( Name = col_character(), IP = col_character(), cl = col_character(), type = col_character(), newA = col_character(), newB = col_character(), oldA = col_character(), oldB = col_character() ) ) %>% filter(cl %in% productCls) %>% gather(key = "cntName", value = "cntIP", 5:6) ####    VNX #### #         (      ) # NaviSECCli.exe -scope 0 -h 10.201.16.15 -user root -password Secrt4yo getlog -date 07/16/2019 07/17/2019 #   ,   . #      ,       errorNumbers <- c("\\(820\\)", "\\(803\\)", "\\(801\\)", "\\(920\\)", "\\(901\\)") ##   ## #         numCores <- detectCores() registerDoParallel(numCores) #    ,   NaviCLI #  1    ,     "" , # ..   ,      ,     . bigPeriodForm <- bigPeriod %>% format(format = "%m/%d/%Y") currDateForm <- (Sys.Date() + 1) %>% format(format = "%m/%d/%Y") #    . #      VNX   ip   . #      ,      #   ,      dataframe #  %dopar%   . #    ,   system.time   , #    %dopar%  %do%.      4-5. # system.time({ errors_df <- foreach(i = 1:nrow(VNX_ip), .combine = rbind, .packages = "tidyverse") %dopar% { errors_raw <- system( paste( "NaviSECCli.exe -scope 0 -h", VNX_ip$cntIP[i], "-user myusername -password mypassword getlog -date", bigPeriodForm, currDateForm ), intern = TRUE ) %>% str_subset(pattern = regex(paste0(errorNumbers, collapse = "|"))) #      ,       if (length(errors_raw) > 0) { #       , #     , #       . errorsDescr <- errors_raw %>% gsub("(.*\\) )(.*)(\\s+\\[.*)", "\\2", x = .) %>% trimws() %>% gsub('([[:punct:]])|\\s+', '_', .) #           errors <- errors_raw %>% str_split(pattern = "\\s+", simplify = T) %>% as_tibble() %>% mutate(Disk = paste0(V4, "_", V6, "_", V8) %>% gsub( pattern = "\\([0-9]{3}\\)", replacement = "", x = .) ) #  dataframe    data_frame(cl = VNX_ip$cl[i], Storage = VNX_ip$Name[i], Date = errors$V1 %>% as.Date(format = "%m/%d/%Y"), Time = errors$V2, Disk = errors$Disk, Error = errorsDescr, eCode = errors$V8 %>% str_extract(paste0(errorNumbers, collapse = "|")) %>% str_extract("[0-9]+")) %>% mutate(DateTime = as.POSIXct(paste(Date, Time), format = "%Y-%m-%d %H:%M:%S")) } } # }) #     .      ,    . stopImplicitCluster() ####  #### #    .    ,     .  . # https://dplyr.tidyverse.org/articles/programming.html myErrorStats <- function(data, period, orderColname = quo(Soft_Media_Error)) { data %>% filter(Date > period) %>% group_by(cl, Storage, Disk, Error) %>% summarise(count = n()) %>% spread(Error, count, fill = 0) %>% arrange(desc(!!orderColname)) } #           .  . errorsBigPeriod <- errors_df %>% myErrorStats(bigPeriod) #             errorsSmallPeriod <- errors_df %>% myErrorStats(smallPeriod) #   ,     errors_filePath <- file.path("results", "VNX_Errors.xlsx") ####  xlsx  #### #    wb<-createWorkbook(type="xlsx") #         TABLE_ROWNAMES_STYLE <- CellStyle(wb) + Font(wb, isBold=TRUE) TABLE_COLNAMES_STYLE <- CellStyle(wb) + Font(wb, isBold=TRUE) + Alignment(wrapText=TRUE, horizontal="ALIGN_CENTER") + Border(color="black", position=c("TOP", "BOTTOM"), pen=c("BORDER_THIN", "BORDER_THICK")) #  t s sheetSmall <- createSheet(wb, sheetName = " 3 ") sheetBig <- createSheet(wb, sheetName = " ") sheetRaw <- createSheet(wb, sheetName = " ") ##   addDataFrame( errorsSmallPeriod %>% as.data.frame(), sheetSmall, startRow = 1, startColumn = 1, row.names = FALSE, byrow = FALSE, colnamesStyle = TABLE_COLNAMES_STYLE, rownamesStyle = TABLE_ROWNAMES_STYLE ) addDataFrame( errorsBigPeriod %>% as.data.frame(), sheetBig, startRow = 1, startColumn = 1, row.names = FALSE, byrow = FALSE, colnamesStyle = TABLE_COLNAMES_STYLE, rownamesStyle = TABLE_ROWNAMES_STYLE ) #          DateTime     addDataFrame( errors_df %>% as.data.frame() %>% arrange(desc(DateTime)), sheetRaw, startRow = 1, startColumn = 1, row.names = FALSE, byrow = FALSE, colnamesStyle = TABLE_COLNAMES_STYLE, rownamesStyle = TABLE_ROWNAMES_STYLE ) #  ,     autoSizeColumn(sheet = sheetSmall, colIndex=c(1:ncol(errorsSmallPeriod))) autoSizeColumn(sheet = sheetBig, colIndex=c(1:ncol(errorsBigPeriod))) autoSizeColumn(sheet = sheetRaw, colIndex=c(1:ncol(errors_df))) #     .   - . if (file.exists(errors_filePath)) {file.remove(errors_filePath)} #  xlsx  saveWorkbook(wb, errors_filePath) ####   #### #       errorsTotal <- errors_df %>% group_by(Date, Error) %>% summarise(count = n()) %>% # spread(Error, count, fill = 0) %>% arrange(desc(Date)) #         plot <- errorsTotal %>% ggplot(aes(x = Date, y = count, fill = Error)) + geom_bar(stat = "identity", width = 0.5, color = "grey") + theme_minimal() + # theme(legend.position="top") + scale_color_grey() + labs(title = "  EMC VNX", subtitle = "        ", fill = " ") + xlab("") + ylab("   ") + scale_fill_brewer(palette = "Spectral") + facet_grid(rows = vars(factor( Error, levels = c( "Soft_SCSI_Bus_Error", "Soft_Media_Error", "Hard_SCSI_Bus_Error", "Recommend_Disk_Replacement" ) ))) # #   # plot #   png,     plot_filePath <- file.path("results", "plot.png") #    png     ggsave(filename = plot_filePath, plot = plot) ####    #### #    emailRecepientsList <- c("sendall-tech@domain.ru") #      emailParams <- list( from = "login@domain.ru", to = emailRecepientsList, smtpParams = list( host.name = "10.10.10.1", port = 25, user.name = "login@domain.ru", passwd = "mypassword", ssl = FALSE ) ) #     (    ). #     ,          ,     . errorsTotal <- errorsSmallPeriod[-c(1,2,3)] %>% sum() #    emailBody <- paste0( '<html> <h3> ,  .</h3> <p>  3    <strong>', errorsTotal, '</strong>    EMC VNX</p> <p>       ,      .</p> <p>  3 : <ul> <li>   3 .   <strong>Soft_Media_Error</strong>.</li> <li>  30 .   <strong>Soft_Media_Error</strong>.</li> <li> .   <strong></strong>.</li> </ul>          .   .</p> <p><img src="', plot_filePath, '"></p> </html>' ) ####      #### send.mail(from = emailParams$from, to = emailParams$to, subject = "     EMC VNX", body = emailBody, encoding = "utf-8", html = TRUE, inline = TRUE, smtp = emailParams$smtpParams, authenticate = TRUE, send = TRUE, attach.files = c(errors_filePath), debug = FALSE) 

 > sessionInfo() R version 3.5.3 (2019-03-11) Platform: x86_64-w64-mingw32/x64 (64-bit) Running under: Windows Server 2012 R2 x64 (build 9600) Matrix products: default locale: [1] LC_COLLATE=Russian_Russia.1251 LC_CTYPE=Russian_Russia.1251 LC_MONETARY=Russian_Russia.1251 [4] LC_NUMERIC=C LC_TIME=Russian_Russia.1251 attached base packages: [1] parallel stats graphics grDevices utils datasets methods base other attached packages: [1] taskscheduleR_1.4 pander_0.6.3 doParallel_1.0.14 iterators_1.0.10 foreach_1.4.4 mailR_0.4.1 [7] xlsx_0.6.1 stringi_1.4.3 zoo_1.8-6 lubridate_1.7.4 wesanderson_0.3.6 forcats_0.4.0 [13] stringr_1.4.0 dplyr_0.8.3 purrr_0.3.2 readr_1.3.1 tidyr_0.8.3 tibble_2.1.3 [19] ggplot2_3.2.0 tidyverse_1.2.1 loaded via a namespace (and not attached): [1] tidyselect_0.2.5 reshape2_1.4.3 rJava_0.9-11 haven_2.1.1 lattice_0.20-38 colorspace_1.4-1 [7] vctrs_0.2.0 generics_0.0.2 utf8_1.1.4 rlang_0.4.0 R.oo_1.22.0 pillar_1.4.2 [13] glue_1.3.1 withr_2.1.2 R.utils_2.9.0 RColorBrewer_1.1-2 modelr_0.1.4 readxl_1.3.1 [19] plyr_1.8.4 munsell_0.5.0 gtable_0.3.0 cellranger_1.1.0 rvest_0.3.4 R.methodsS3_1.7.1 [25] codetools_0.2-16 labeling_0.3 fansi_0.4.0 xlsxjars_0.6.1 broom_0.5.2 Rcpp_1.0.1 [31] scales_1.0.0 backports_1.1.4 jsonlite_1.6 digest_0.6.20 hms_0.5.0 grid_3.5.3 [37] cli_1.1.0 tools_3.5.3 magrittr_1.5 lazyeval_0.2.2 crayon_1.3.4 pkgconfig_2.0.2 [43] zeallot_0.1.0 data.table_1.12.2 xml2_1.2.0 assertthat_0.2.1 httr_1.4.0 rstudioapi_0.10 [49] R6_2.4.0 nlme_3.1-137 compiler_3.5.3