Merge pull request 'Enhancements to blockchain scanner' (#74) from rav001 into dev041

Reviewed-on: #74

app/ZenScan.hs

@@ -8,8 +8,7 @@ import Zenith.Scanner (scanZebra)
 main :: IO ()
 main = do
   config <- load ["zenith.cfg"]
-  --dbFilePath <- require config "dbFilePath"
+  dbFilePath <- require config "dbFilePath"
-  dataStorePath <- require config "dataStorePath"
   zebraPort <- require config "zebraPort"
   zebraHost <- require config "zebraHost"
-  scanZebra 2781518 zebraHost zebraPort dataStorePath
+  scanZebra 2762066 zebraHost zebraPort dbFilePath

src/Zenith/Core.hs

@@ -157,3 +157,11 @@ createWalletAddress n i zNet scope za = do
       (UnifiedAddressDB $
        encodeUnifiedAddress $ UnifiedAddress zNet oRec sRec (Just tRec))
       (ScopeDB scope)
+
+-- * Wallet
+-- | Sync the wallet with the data store
+syncWallet ::
+     T.Text -- ^ The database path
+  -> Entity ZcashWallet
+  -> IO ()
+syncWallet walletDb w = undefined

src/Zenith/DB.hs

@@ -78,23 +78,49 @@ share
     UniqueAddress index scope accId
     UniqueAddName accId name
     deriving Show Eq
-    |]
-
-share
-  [mkPersist sqlSettings, mkMigrate "rawStorage"]
-  [persistLowerCase|
   WalletTransaction
+    txId HexStringDB
+    block Int
+    conf Int
+    time Int
+    deriving Show Eq
+  WalletTrNote
+    tx WalletTransactionId
+    addrId WalletAddressId
+    value Int
+    rawId TransparentNoteId
+    spent Bool
+    deriving Show Eq
+  WalletSapNote
+    tx WalletTransactionId
+    addrId WalletAddressId
+    value Int
+    recipient BS.ByteString
+    memo T.Text
+    rawId ShieldOutputId
+    spent Bool
+    deriving Show Eq
+  WalletOrchNote
+    tx WalletTransactionId
+    addrId WalletAddressId
+    value Int
+    recipient BS.ByteString
+    memo T.Text
+    rawId OrchActionId
+    spent Bool
+    deriving Show Eq
+  ZcashTransaction
     block Int
     txId HexStringDB
     conf Int
     time Int
     deriving Show Eq
   TransparentNote
-    tx WalletTransactionId
+    tx ZcashTransactionId
     value Int
     script BS.ByteString
   OrchAction
-    tx WalletTransactionId
+    tx ZcashTransactionId
     nf HexStringDB
     rk HexStringDB
     cmx HexStringDB
@@ -105,7 +131,7 @@ share
     auth HexStringDB
     deriving Show Eq
   ShieldOutput
-    tx WalletTransactionId
+    tx ZcashTransactionId
     cv HexStringDB
     cmu HexStringDB
     ephKey HexStringDB
@@ -114,7 +140,7 @@ share
     proof HexStringDB
     deriving Show Eq
   ShieldSpend
-    tx WalletTransactionId
+    tx ZcashTransactionId
     cv HexStringDB
     anchor HexStringDB
     nullifier HexStringDB
@@ -132,12 +158,6 @@ initDb ::
 initDb dbName = do
   runSqlite dbName $ do runMigration migrateAll
 
--- | Initializes the raw data storage
-initRawStore ::
-     T.Text -- ^ the database path
-  -> IO ()
-initRawStore dbFilePath = runSqlite dbFilePath $ runMigration rawStorage
-
 -- | Get existing wallets from database
 getWallets :: T.Text -> ZcashNet -> IO [Entity ZcashWallet]
 getWallets dbFp n =
@@ -177,6 +197,30 @@ saveAccount ::
   -> IO (Maybe (Entity ZcashAccount))
 saveAccount dbFp a = runSqlite dbFp $ insertUniqueEntity a
 
+-- | Returns the largest block in storage
+getMaxBlock ::
+     T.Text -- ^ The database path
+  -> IO Int
+getMaxBlock dbPath = do
+  b <-
+    runSqlite dbPath $
+    selectFirst [ZcashTransactionBlock >. 0] [Desc ZcashTransactionBlock]
+  case b of
+    Nothing -> return $ -1
+    Just x -> return $ zcashTransactionBlock $ entityVal x
+
+-- | Returns the largest block in the wallet
+getMaxWalletBlock ::
+     T.Text -- ^ The database path
+  -> IO Int
+getMaxWalletBlock dbPath = do
+  b <-
+    runSqlite dbPath $
+    selectFirst [WalletTransactionBlock >. 0] [Desc WalletTransactionBlock]
+  case b of
+    Nothing -> return $ -1
+    Just x -> return $ walletTransactionBlock $ entityVal x
+
 -- | Returns a list of addresses associated with the given account
 getAddresses ::
      T.Text -- ^ The database path
@@ -216,12 +260,12 @@ saveTransaction ::
      T.Text -- ^ the database path
   -> Int -- ^ block time
   -> Transaction -- ^ The transaction to save
-  -> IO (Key WalletTransaction)
+  -> IO (Key ZcashTransaction)
 saveTransaction dbFp t wt =
   runSqlite dbFp $ do
     w <-
       insert $
-      WalletTransaction (tx_height wt) (HexStringDB $ tx_id wt) (tx_conf wt) t
+      ZcashTransaction (tx_height wt) (HexStringDB $ tx_id wt) (tx_conf wt) t
     when (isJust $ tx_transpBundle wt) $
       insertMany_ $
       map (storeTxOut w) $ (tb_vout . fromJust . tx_transpBundle) wt
@@ -238,9 +282,9 @@ saveTransaction dbFp t wt =
       map (storeOrchAction w) $ (obActions . fromJust . tx_orchardBundle) wt
     return w
   where
-    storeTxOut :: WalletTransactionId -> TxOut -> TransparentNote
+    storeTxOut :: ZcashTransactionId -> TxOut -> TransparentNote
     storeTxOut wid (TxOut v s) = TransparentNote wid (fromIntegral v) s
-    storeSapSpend :: WalletTransactionId -> ShieldedSpend -> ShieldSpend
+    storeSapSpend :: ZcashTransactionId -> ShieldedSpend -> ShieldSpend
     storeSapSpend wid sp =
       ShieldSpend
         wid
@@ -250,7 +294,7 @@ saveTransaction dbFp t wt =
         (HexStringDB $ sp_rk sp)
         (HexStringDB $ sp_proof sp)
         (HexStringDB $ sp_auth sp)
-    storeSapOutput :: WalletTransactionId -> ShieldedOutput -> ShieldOutput
+    storeSapOutput :: ZcashTransactionId -> ShieldedOutput -> ShieldOutput
     storeSapOutput wid so =
       ShieldOutput
         wid
@@ -260,7 +304,7 @@ saveTransaction dbFp t wt =
         (HexStringDB $ s_encCipherText so)
         (HexStringDB $ s_outCipherText so)
         (HexStringDB $ s_proof so)
-    storeOrchAction :: WalletTransactionId -> OrchardAction -> OrchAction
+    storeOrchAction :: ZcashTransactionId -> OrchardAction -> OrchAction
     storeOrchAction wid oa =
       OrchAction
         wid

src/Zenith/Scanner.hs

@@ -8,6 +8,7 @@ import Data.HexString
 import Data.Maybe
 import qualified Data.Text as T
 import GHC.Utils.Monad (concatMapM)
+import System.Console.AsciiProgress
 import ZcashHaskell.Types
   ( BlockResponse(..)
   , RawZebraTx(..)
@@ -20,7 +21,7 @@ import ZcashHaskell.Types
   )
 import ZcashHaskell.Utils (getBlockTime, makeZebraCall, readZebraTransaction)
 import Zenith.Core (checkBlockChain)
-import Zenith.DB (initRawStore, saveTransaction)
+import Zenith.DB (getMaxBlock, initDb, saveTransaction)
 import Zenith.Utils (jsonNumber)
 
 -- | Function to scan the Zcash blockchain through the Zebra node and populate the Zenith database
@@ -31,32 +32,37 @@ scanZebra ::
   -> T.Text -- ^ Path to database file
   -> IO ()
 scanZebra b host port dbFilePath = do
-  _ <- initRawStore dbFilePath
+  _ <- initDb dbFilePath
   bc <-
     try $ checkBlockChain host port :: IO
       (Either IOError ZebraGetBlockChainInfo)
   case bc of
     Left e -> print e
     Right bStatus -> do
-      if b > zgb_blocks bStatus || b < 1
+      dbBlock <- getMaxBlock dbFilePath
+      let sb = max dbBlock b
+      if sb > zgb_blocks bStatus || sb < 1
         then throwIO $ userError "Invalid starting block for scan"
         else do
-          let bList = [b .. (zgb_blocks bStatus)]
+          let bList = [sb .. (zgb_blocks bStatus)]
-          txList <-
+          displayConsoleRegions $ do
-            try $ mapM_ (processBlock host port dbFilePath) bList :: IO
+            pg <- newProgressBar def {pgTotal = fromIntegral $ length bList}
-              (Either IOError ())
+            txList <-
-          case txList of
+              try $ mapM_ (processBlock host port dbFilePath pg) bList :: IO
-            Left e1 -> print e1
+                (Either IOError ())
-            Right txList' -> print txList'
+            case txList of
+              Left e1 -> print e1
+              Right txList' -> print txList'
 
 -- | Function to process a raw block and extract the transaction information
 processBlock ::
      T.Text -- ^ Host name for `zebrad`
   -> Int -- ^ Port for `zebrad`
   -> T.Text -- ^ DB file path
+  -> ProgressBar -- ^ Progress bar
   -> Int -- ^ The block number to process
   -> IO ()
-processBlock host port dbFp b = do
+processBlock host port dbFp pg b = do
   r <-
     makeZebraCall
       host
@@ -78,6 +84,7 @@ processBlock host port dbFp b = do
           let blockTime = getBlockTime hb
           mapM_ (processTx host port blockTime dbFp) $
             bl_txs $ addTime blk blockTime
+          tick pg
   where
     addTime :: BlockResponse -> Int -> BlockResponse
     addTime bl t =
@@ -108,7 +115,7 @@ processTx host port bt dbFp t = do
       case readZebraTransaction (ztr_hex rawTx) of
         Nothing -> return ()
         Just rzt -> do
-          k <-
+          _ <-
             saveTransaction dbFp bt $
             Transaction
               t
@@ -118,4 +125,4 @@ processTx host port bt dbFp t = do
               (fromRawTBundle $ zt_tBundle rzt)
               (fromRawSBundle $ zt_sBundle rzt)
               (fromRawOBundle $ zt_oBundle rzt)
-          print k
+          return ()

zenith.cabal

@@ -65,6 +65,7 @@ library
     , vector
     , vty
     , word-wrap
+    , ascii-progress
     , zcash-haskell
   --pkgconfig-depends: rustzcash_wrapper
   default-language: Haskell2010

zenith.cfg

@@ -3,4 +3,3 @@ nodePwd = "superSecret"
 dbFilePath = "zenith.db"
 zebraHost = "127.0.0.1"
 zebraPort = 18232
-dataStorePath = "datastore.db"